添加镜像仓库代码

website/consts.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 """静态配置"""
 
+PAGE_SIZE = 10
+
 # 行业分类
 industry_map = {
     1001: u"IT服务",

website/handlers/file/handler.py

@@ -4,6 +4,7 @@ import logging
 import hashlib
 import re
 import os
+import aiofiles
 from sqlalchemy import text
 
 from website import errors
@@ -13,7 +14,7 @@ from website.handler import APIHandler, authenticated
 
 class UploadHandler(APIHandler):
     @authenticated
-    def post(self):
+    async def post(self):
         file_metas = self.request.files.get('file', None)
         if not file_metas:
             raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "请选择文件")
@@ -47,8 +48,8 @@ class UploadHandler(APIHandler):
                 if not os.path.exists(filepath):
                     for meta in file_metas:
                         # filename = meta['filename']
-                        with open(filepath, 'wb') as f:
-                            f.write(meta['body'])
+                        async with open(filepath, 'wb') as f:
+                            await f.write(meta['body'])
                 
                 sql_insert = text("insert into files(filename, filepath, md5_str, filesize, filetype, user) values(:filename, :filepath, :md5_str, :file_size, :filetype, :user)")
                 conn.execute(sql_insert, {"filename": filename, "filepath": filepath, "md5_str": md5_str, "file_size": int(file_size/1024/1024), "filetype": filetype, "user": self.current_user.id})
@@ -85,6 +86,38 @@ class DeleteHandler(APIHandler):
 
 
 class BigFileUploadHandler(APIHandler):
-    def post(self):
+    async def post(self):
+        file_metas = self.request.files.get('file', None)
+        if not file_metas:
+            raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "请选择文件")
+
+        filename = file_metas[0].filename
+        punctuation = """!"#$%&'()*+,/:;<=>?@[\]^`{|}~ """
+        regex = re.compile('[%s]' % re.escape(punctuation))
+        filename = regex.sub("", filename.replace('..', ''))
+        file_size = len(file_metas[0].body)
+
+        logging.info("file_size: %s", file_size)
+
+        if file_size > 300 * 1024 * 1024:
+            raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, 'Exceed 300M size limit')
+
+        filetype = filename.split(".") and filename.split(".")[-1] or ""
+
+        file_upload_dir = settings.file_upload_dir
+        os.makedirs(file_upload_dir, exist_ok=True)
+
+        # md5_str = hashlib.md5(file_metas[0].body).hexdigest()
+
+        filepath = os.path.join(settings.file_upload_dir,  filename)
+        if not os.path.exists(filepath):
+            for meta in file_metas:
+                # filename = meta['filename']
+                # with open(filepath, 'wb') as f:
+                #     f.write(meta['body'])
+
+                async with aiofiles.open(filepath, 'wb') as f:
+                    await f.write(meta['body'])
 
         self.finish()
+

website/handlers/model/handler.py

@@ -100,7 +100,7 @@ class ListHandler(APIHandler):
     @authenticated
     def post(self):
         pageNo = self.get_int_argument("pageNo", 1)
-        pageSize = self.get_int_argument("pageSize", 10)
+        pageSize = self.get_int_argument("pageSize", consts.PAGE_SIZE)
         name = self.get_escaped_argument("name", "")
 
         result = []

website/handlers/model_hub/handler.py

@@ -0,0 +1,231 @@
+# -*- coding: utf-8 -*-
+import logging
+import os
+import requests
+
+from sqlalchemy import text
+
+from website import consts
+from website import db
+from website import errors
+from website import settings
+from website.handler import APIHandler, authenticated
+
+class ListHandler(APIHandler):
+    """
+    - 描述： 模型运行库列表
+    - 请求方式：post
+    - 请求参数：
+    > - pageNo, int
+    > - pageSize, int
+    > - name, string, 名称
+    - 返回值：
+    ```
+        {
+            "count": 123,
+            "data": [
+                {
+                    "id": 123,
+                    "name": "xx",
+                    "create_time": "xxx",
+                    "update_time": "xxx"
+                },
+                ...
+            ]
+        }
+    ```
+    """
+    @authenticated
+    def post(self):
+        pageNo = self.get_int_argument("pageNo", 1)
+        pageSize = self.get_int_argument("pageSize", consts.PAGE_SIZE)
+        name = self.get_escaped_argument("name", "")
+        
+        result = []
+        count = 0
+        
+        with self.app_mysql.connect() as conn:
+            sql = "select id, name, create_at, update_at from model_hub where 1=1"
+            param = {}
+
+            sql_count = "select count(id) from model where 1=1"
+            param_count = {}
+
+            if name:
+                sql += "and m.name like :name"
+                param["name"] = "%{}%".format(name)
+
+                sql_count += "and m.name like :name"
+                param_count["name"] = "%{}%".format(name)
+
+            sql += " order by m.id desc limit :pageSize offset :offset"
+            param["pageSize"] = pageSize
+            param["offset"] = (pageNo - 1) * pageSize
+
+            cur = conn.execute(text(sql), param)
+            result = db.to_json_list(cur)
+
+            count = conn.execute(text(sql_count), param_count).fetchone()[0]
+
+        data = []
+        for item in result:
+            data.append({
+                "id": item["id"],
+                "name": item["name"],
+                "create_time": item["create_at"].strftime("%Y-%m-%d %H:%M:%S"),
+                "update_time": item["update_at"].strftime("%Y-%m-%d %H:%M:%S")
+            })
+
+        self.finish({"count": count, "data": data})
+        
+
+
+class SyncHandler(APIHandler):
+    """
+    - 描述： 查询docker registry中的镜像
+    - 请求方式：post
+    - 请求参数：
+    > - host, string, ip地址
+    > - port, int, 端口
+    - 返回值：
+    ```
+        {
+            "data": [
+                "xxx",    # docker registry中docker images的地址
+                "xxx",
+                ...
+            ]
+        }
+```
+    """
+    @authenticated
+    def post(self):
+        host = self.get_escaped_argument("host", "")
+        port = self.get_int_argument("port")
+        if not host or not port:
+            raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "host and port must be provided.")
+        
+        images = []
+        # 查询docker registry中的镜像
+        repositories = requests.get("http://{}:{}/v2/_catalog".format(host, port)).json()["repositories"]
+        for repository in repositories:
+            # 查询docker registry中的镜像的tag
+            tags = requests.get("http://{}:{}/v2/{}/tags/list".format(host, port, repository)).json()["tags"]
+            for tag in tags:
+                image_name = "{}:{}/{}:{}".format(host, port, repository, tag)
+                images.append(image_name)
+        self.finish({"data": images})
+
+
+
+class AddHandler(APIHandler):
+    """
+    - 描述： 新建模型运行库
+    - 请求方式：post
+    - 请求参数：
+    > - name, string, 名称
+    > - host, string,
+    > - port, int
+    > - path, string, 镜像路径
+    > - comment, string, 备注
+    - 返回值：无
+    """
+    @authenticated
+    def post(self):
+        name = self.get_escaped_argument("name", "")
+        host = self.get_escaped_argument("host", "")
+        port = self.get_int_argument("port")
+        path = self.get_escaped_argument("path", "")
+        comment = self.get_escaped_argument("comment", "")
+        if not name or not host or not port:
+            raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "name and host and port must be provided.")
+        
+        with self.app_mysql.connect() as conn:
+            conn.execute(text("""insert into model_hub (name, host, port, path, comment, create_at, update_at) 
+                              values (:name, :host, :port, :path, :comment, NOW(), NOW())"""),
+                          {"name": name, "host": host, "port": port, "path": path, "comment": comment})
+
+            conn.commit()
+
+        self.finish()
+
+
+class EditHandler(APIHandler):
+    """
+    - 描述： 编辑模型运行库
+    - 请求方式：post
+    - 请求参数：
+    > - id, int
+    > - name, string, 名称
+    > - host, string,
+    > - port, int
+    > - path, string, 镜像路径
+    > - comment, string, 备注
+    - 返回值：无
+    """
+    @authenticated
+    def post(self):
+        id = self.get_int_argument("id")
+        name = self.get_escaped_argument("name", "")
+        host = self.get_escaped_argument("host", "")
+        port = self.get_int_argument("port")
+        path = self.get_escaped_argument("path", "")
+        comment = self.get_escaped_argument("comment", "")
+        if not id or not name or not host or not port or path:
+            raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "parameter error")
+        with self.app_mysql.connect() as conn:
+            conn.execute(text("""update model_hub set name=:name, host=:host, port=:port, path=:path, comment=:comment, update_at=NOW() 
+                              where id=:id"""), {"id": id, "name": name, "host": host, "port": port, "path": path, "comment": comment})
+            conn.commit()
+        self.finish()
+
+
+class InfoHandler(APIHandler):
+    """
+    - 描述： 模型运行库信息
+    - 请求方式：post
+    - 请求参数：
+    > - id, int
+    - 返回值：
+    ```
+    {
+        "name": "xxx", 
+        "host": "xxx", 
+        "port": 123, 
+        "path": "xxx", 
+        "comment": "xxx", 
+    }
+    ```
+    """
+    @authenticated
+    def post(self):
+        hid = self.get_int_argument("id")
+        if not id:
+            raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "parameter error")
+
+        result = {}
+        with self.app_mysql.connect() as conn:
+            cur = conn.execute(text("""select name, host, port, path, comment from model_hub where id=:id"""), {"id": hid})
+            result = db.to_json(cur)
+            if not result:
+                raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "model hub not found")
+        self.finish({"data": result})
+
+
+class DeleteHandler(APIHandler):
+    """
+    - 描述： 删除模型运行库
+    - 请求方式：post
+    - 请求参数：
+    > - id, int
+    - 返回值：无
+    """
+    @authenticated
+    def post(self):
+        hid = self.get_int_argument("id")
+        if not id:
+            raise errors.HTTPAPIError(errors.ERROR_BAD_REQUEST, "parameter error")
+        with self.app_mysql.connect() as conn:
+            conn.execute(text("""delete from model_hub where id=:id"""), {"id": hid})
+            conn.commit()
+        self.finish()

website/handlers/model_hub/url.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+from website.handlers.model_hub import handler
+
+handlers = [
+# ("/", handler.Handler),
+    ("/model/hub/list", handler.ListHandler),
+    ("/model/hub/sync", handler.SyncHandler),
+    ("/model/hub/add", handler.AddHandler),
+    ("/model/hub/edit", handler.EditHandler),
+    ("/model/hub/info", handler.InfoHandler),
+    ("/model/hub/delete", handler.DeleteHandler),
+]
+
+page_handlers = [
+]