diff --git a/website/consts.py b/website/consts.py index 971885d..3c485d6 100644 --- a/website/consts.py +++ b/website/consts.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- """静态配置""" +PAGE_SIZE = 10 + # 行业分类 industry_map = { 1001: u"IT服务", diff --git a/website/handlers/file/handler.py b/website/handlers/file/handler.py index 7560a5c..632ba13 100644 --- a/website/handlers/file/handler.py +++ b/website/handlers/file/handler.py @@ -4,6 +4,7 @@ import logging import hashlib import re import os +import aiofiles from sqlalchemy import text from website import errors @@ -13,7 +14,7 @@ from website.handler import APIHandler, authenticated class UploadHandler(APIHandler): @authenticated - def post(self): + async def post(self): file_metas = self.request.files.get('file', None) if not file_metas: raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "请选择文件") @@ -47,8 +48,8 @@ class UploadHandler(APIHandler): if not os.path.exists(filepath): for meta in file_metas: # filename = meta['filename'] - with open(filepath, 'wb') as f: - f.write(meta['body']) + async with open(filepath, 'wb') as f: + await f.write(meta['body']) sql_insert = text("insert into files(filename, filepath, md5_str, filesize, filetype, user) values(:filename, :filepath, :md5_str, :file_size, :filetype, :user)") conn.execute(sql_insert, {"filename": filename, "filepath": filepath, "md5_str": md5_str, "file_size": int(file_size/1024/1024), "filetype": filetype, "user": self.current_user.id}) @@ -85,6 +86,38 @@ class DeleteHandler(APIHandler): class BigFileUploadHandler(APIHandler): - def post(self): - - self.finish() \ No newline at end of file + async def post(self): + file_metas = self.request.files.get('file', None) + if not file_metas: + raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "请选择文件") + + filename = file_metas[0].filename + punctuation = """!"#$%&'()*+,/:;<=>?@[\]^`{|}~ """ + regex = re.compile('[%s]' % re.escape(punctuation)) + filename = regex.sub("", filename.replace('..', '')) + file_size = len(file_metas[0].body) + + logging.info("file_size: %s", file_size) + + if file_size > 300 * 1024 * 1024: + raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, 'Exceed 300M size limit') + + filetype = filename.split(".") and filename.split(".")[-1] or "" + + file_upload_dir = settings.file_upload_dir + os.makedirs(file_upload_dir, exist_ok=True) + + # md5_str = hashlib.md5(file_metas[0].body).hexdigest() + + filepath = os.path.join(settings.file_upload_dir, filename) + if not os.path.exists(filepath): + for meta in file_metas: + # filename = meta['filename'] + # with open(filepath, 'wb') as f: + # f.write(meta['body']) + + async with aiofiles.open(filepath, 'wb') as f: + await f.write(meta['body']) + + self.finish() + diff --git a/website/handlers/model/handler.py b/website/handlers/model/handler.py index 1850e77..46e70fa 100644 --- a/website/handlers/model/handler.py +++ b/website/handlers/model/handler.py @@ -100,7 +100,7 @@ class ListHandler(APIHandler): @authenticated def post(self): pageNo = self.get_int_argument("pageNo", 1) - pageSize = self.get_int_argument("pageSize", 10) + pageSize = self.get_int_argument("pageSize", consts.PAGE_SIZE) name = self.get_escaped_argument("name", "") result = [] diff --git a/website/handlers/model_hub/__init__.py b/website/handlers/model_hub/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/website/handlers/model_hub/handler.py b/website/handlers/model_hub/handler.py new file mode 100644 index 0000000..d549d42 --- /dev/null +++ b/website/handlers/model_hub/handler.py @@ -0,0 +1,231 @@ +# -*- coding: utf-8 -*- +import logging +import os +import requests + +from sqlalchemy import text + +from website import consts +from website import db +from website import errors +from website import settings +from website.handler import APIHandler, authenticated + +class ListHandler(APIHandler): + """ + - 描述: 模型运行库列表 + - 请求方式:post + - 请求参数: + > - pageNo, int + > - pageSize, int + > - name, string, 名称 + - 返回值: + ``` + { + "count": 123, + "data": [ + { + "id": 123, + "name": "xx", + "create_time": "xxx", + "update_time": "xxx" + }, + ... + ] + } + ``` + """ + @authenticated + def post(self): + pageNo = self.get_int_argument("pageNo", 1) + pageSize = self.get_int_argument("pageSize", consts.PAGE_SIZE) + name = self.get_escaped_argument("name", "") + + result = [] + count = 0 + + with self.app_mysql.connect() as conn: + sql = "select id, name, create_at, update_at from model_hub where 1=1" + param = {} + + sql_count = "select count(id) from model where 1=1" + param_count = {} + + if name: + sql += "and m.name like :name" + param["name"] = "%{}%".format(name) + + sql_count += "and m.name like :name" + param_count["name"] = "%{}%".format(name) + + sql += " order by m.id desc limit :pageSize offset :offset" + param["pageSize"] = pageSize + param["offset"] = (pageNo - 1) * pageSize + + cur = conn.execute(text(sql), param) + result = db.to_json_list(cur) + + count = conn.execute(text(sql_count), param_count).fetchone()[0] + + data = [] + for item in result: + data.append({ + "id": item["id"], + "name": item["name"], + "create_time": item["create_at"].strftime("%Y-%m-%d %H:%M:%S"), + "update_time": item["update_at"].strftime("%Y-%m-%d %H:%M:%S") + }) + + self.finish({"count": count, "data": data}) + + + +class SyncHandler(APIHandler): + """ + - 描述: 查询docker registry中的镜像 + - 请求方式:post + - 请求参数: + > - host, string, ip地址 + > - port, int, 端口 + - 返回值: + ``` + { + "data": [ + "xxx", # docker registry中docker images的地址 + "xxx", + ... + ] + } +``` + """ + @authenticated + def post(self): + host = self.get_escaped_argument("host", "") + port = self.get_int_argument("port") + if not host or not port: + raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "host and port must be provided.") + + images = [] + # 查询docker registry中的镜像 + repositories = requests.get("http://{}:{}/v2/_catalog".format(host, port)).json()["repositories"] + for repository in repositories: + # 查询docker registry中的镜像的tag + tags = requests.get("http://{}:{}/v2/{}/tags/list".format(host, port, repository)).json()["tags"] + for tag in tags: + image_name = "{}:{}/{}:{}".format(host, port, repository, tag) + images.append(image_name) + self.finish({"data": images}) + + + +class AddHandler(APIHandler): + """ + - 描述: 新建模型运行库 + - 请求方式:post + - 请求参数: + > - name, string, 名称 + > - host, string, + > - port, int + > - path, string, 镜像路径 + > - comment, string, 备注 + - 返回值:无 + """ + @authenticated + def post(self): + name = self.get_escaped_argument("name", "") + host = self.get_escaped_argument("host", "") + port = self.get_int_argument("port") + path = self.get_escaped_argument("path", "") + comment = self.get_escaped_argument("comment", "") + if not name or not host or not port: + raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "name and host and port must be provided.") + + with self.app_mysql.connect() as conn: + conn.execute(text("""insert into model_hub (name, host, port, path, comment, create_at, update_at) + values (:name, :host, :port, :path, :comment, NOW(), NOW())"""), + {"name": name, "host": host, "port": port, "path": path, "comment": comment}) + + conn.commit() + + self.finish() + + +class EditHandler(APIHandler): + """ + - 描述: 编辑模型运行库 + - 请求方式:post + - 请求参数: + > - id, int + > - name, string, 名称 + > - host, string, + > - port, int + > - path, string, 镜像路径 + > - comment, string, 备注 + - 返回值:无 + """ + @authenticated + def post(self): + id = self.get_int_argument("id") + name = self.get_escaped_argument("name", "") + host = self.get_escaped_argument("host", "") + port = self.get_int_argument("port") + path = self.get_escaped_argument("path", "") + comment = self.get_escaped_argument("comment", "") + if not id or not name or not host or not port or path: + raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "parameter error") + with self.app_mysql.connect() as conn: + conn.execute(text("""update model_hub set name=:name, host=:host, port=:port, path=:path, comment=:comment, update_at=NOW() + where id=:id"""), {"id": id, "name": name, "host": host, "port": port, "path": path, "comment": comment}) + conn.commit() + self.finish() + + +class InfoHandler(APIHandler): + """ + - 描述: 模型运行库信息 + - 请求方式:post + - 请求参数: + > - id, int + - 返回值: + ``` + { + "name": "xxx", + "host": "xxx", + "port": 123, + "path": "xxx", + "comment": "xxx", + } + ``` + """ + @authenticated + def post(self): + hid = self.get_int_argument("id") + if not id: + raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "parameter error") + + result = {} + with self.app_mysql.connect() as conn: + cur = conn.execute(text("""select name, host, port, path, comment from model_hub where id=:id"""), {"id": hid}) + result = db.to_json(cur) + if not result: + raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "model hub not found") + self.finish({"data": result}) + + +class DeleteHandler(APIHandler): + """ + - 描述: 删除模型运行库 + - 请求方式:post + - 请求参数: + > - id, int + - 返回值:无 + """ + @authenticated + def post(self): + hid = self.get_int_argument("id") + if not id: + raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "parameter error") + with self.app_mysql.connect() as conn: + conn.execute(text("""delete from model_hub where id=:id"""), {"id": hid}) + conn.commit() + self.finish() diff --git a/website/handlers/model_hub/url.py b/website/handlers/model_hub/url.py new file mode 100644 index 0000000..72a60c5 --- /dev/null +++ b/website/handlers/model_hub/url.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from website.handlers.model_hub import handler + +handlers = [ +# ("/", handler.Handler), + ("/model/hub/list", handler.ListHandler), + ("/model/hub/sync", handler.SyncHandler), + ("/model/hub/add", handler.AddHandler), + ("/model/hub/edit", handler.EditHandler), + ("/model/hub/info", handler.InfoHandler), + ("/model/hub/delete", handler.DeleteHandler), +] + +page_handlers = [ +] \ No newline at end of file