[td] 0.7.89 (mage-ai#1938)

.gitignore

@@ -159,3 +159,6 @@ docker-compose.override.yml
 
 # vscode
 .vscode/
+
+# front-end
+node_modules

docs/development/project/setup.mdx

@@ -0,0 +1,66 @@
+---
+title: "Project setup"
+description: "How to set up a folder containing your Mage project code and other files."
+---
+
+## Parent folder
+
+Create a parent folder that will contain your Mage project code:
+
+```bash
+mkdir data_monorepo
+```
+
+Change directory into the parent folder:
+
+```bash
+cd data_monorepo
+```
+
+## Create a Dockerfile
+
+Create an empty Dockerfile:
+
+```bash
+echo "" > Dockerfile
+```
+
+Paste the contents from this [Dockerfile](https://github.com/mage-ai/docker/blob/master/Dockerfile)
+into the Dockerfile you just created.
+
+## Create a .gitignore file
+
+Create an empty .gitignore file:
+
+```bash
+echo "" > .gitignore
+```
+
+Paste the following contents into the .gitignore file you just created:
+
+```
+.DS_Store
+.gitkeep
+.log
+.logs/
+.variables/
+__pycache__/
+docker-compose.override.yml
+logs/
+mage-ai.db
+mage_data/
+secrets/
+```
+
+## Create Mage project
+
+Follow these [instructions](/getting-started/setup) to create a Mage project in your
+current directory. If you named your project `demo_project`, your current folder structure
+will look like this:
+
+```
+data_monorepo/
+|-- demo_project/
+|-- .gitignore
+|-- Dockerfile
+```

docs/mint.json

@@ -128,6 +128,7 @@
     {
       "group": "Development",
       "pages": [
+        "development/project/setup",
         {
           "group": "Pipelines",
           "pages": [

mage_ai/api/middleware.py

@@ -1,7 +1,7 @@
 from json.decoder import JSONDecodeError
 from mage_ai.api.errors import ApiError
 from mage_ai.orchestration.db.models import Oauth2AccessToken, Oauth2Application
-from mage_ai.settings import OAUTH2_APPLICATION_CLIENT_ID
+from mage_ai.settings import OAUTH2_APPLICATION_CLIENT_ID, REQUIRE_USER_AUTHENTICATION
 from mage_ai.server.api.constants import (
     ENDPOINTS_BYPASS_OAUTH_CHECK,
     HEADER_API_KEY,
@@ -19,6 +19,9 @@ def prepare(self):
         self.request.__setattr__('oauth_client', None)
         self.request.__setattr__('oauth_token', None)
 
+        if not REQUIRE_USER_AUTHENTICATION:
+            return
+
         paths = [path for path in self.request.uri.split('/') if path]
         if any(p in ENDPOINTS_BYPASS_OAUTH_CHECK for p in paths):
             return

mage_ai/api/operations/base.py

@@ -7,6 +7,7 @@
     CREATE,
     DELETE,
     DETAIL,
+    FILE_KEY_NAME,
     LIST,
     META_KEY_FORMAT,
     READ,
@@ -218,7 +219,9 @@ def __combined_options(self):
     def __payload_for_resource(self):
         payload = self.payload.get(self.__resource_name_singular(), {})
         if self.files and self.files.get(FILE_KEY_NAME):
+            payload_prev = ignore_keys(self.payload, [FILE_KEY_NAME])
             payload[FILE_KEY_NAME] = self.files.get(FILE_KEY_NAME)
+            payload.update(payload_prev)
         return payload
 
     def __present_results(self, results):

mage_ai/api/policies/FilePolicy.py

@@ -0,0 +1,33 @@
+from mage_ai.api.oauth_scope import OauthScope
+from mage_ai.api.operations import constants
+from mage_ai.api.policies.BasePolicy import BasePolicy
+from mage_ai.api.presenters.FilePresenter import FilePresenter
+
+
+class FilePolicy(BasePolicy):
+    pass
+
+
+FilePolicy.allow_actions([
+    constants.CREATE,
+    constants.LIST,
+], scopes=[
+    OauthScope.CLIENT_PRIVATE,
+])
+
+FilePolicy.allow_read([] + FilePresenter.default_attributes, scopes=[
+    OauthScope.CLIENT_PRIVATE,
+], on_action=[
+    constants.CREATE,
+    constants.LIST,
+])
+
+FilePolicy.allow_write([
+    'dir_path',
+    'name',
+    'overwrite',
+], scopes=[
+    OauthScope.CLIENT_PRIVATE,
+], on_action=[
+    constants.CREATE,
+])

mage_ai/api/presenters/FilePresenter.py

@@ -0,0 +1,10 @@
+from mage_ai.api.presenters.BasePresenter import BasePresenter
+
+
+class FilePresenter(BasePresenter):
+    default_attributes = [
+        'children',
+        'disabled',
+        'name',
+        'path',
+    ]

mage_ai/api/resources/FileResource.py

@@ -0,0 +1,44 @@
+from mage_ai.api.errors import ApiError
+from mage_ai.api.resources.GenericResource import GenericResource
+from mage_ai.data_preparation.models.errors import FileExistsError
+from mage_ai.data_preparation.models.file import File
+from mage_ai.data_preparation.repo_manager import get_repo_path
+from typing import Dict
+
+
+class FileResource(GenericResource):
+    @classmethod
+    def collection(self, query, meta, user, **kwargs):
+        return self.build_result_set(
+            [File.get_all_files(get_repo_path())],
+            user,
+            **kwargs,
+        )
+
+    @classmethod
+    def create(self, payload: Dict, user, **kwargs) -> 'FileResource':
+        dir_path = payload['dir_path']
+        repo_path = get_repo_path()
+        content = None
+
+        if 'file' in payload:
+            file = payload['file'][0]
+            filename = file['filename']
+            content = file['body']
+        else:
+            filename = payload['name']
+
+        try:
+            file = File.create(
+                filename,
+                dir_path,
+                repo_path=repo_path,
+                content=content,
+                overwrite=payload.get('overwrite', False),
+            )
+
+            return self(file.to_dict(), user, **kwargs)
+        except FileExistsError as err:
+            error = ApiError.RESOURCE_INVALID.copy()
+            error.update(dict(message=str(err)))
+            raise ApiError(error)

mage_ai/api/resources/GenericResource.py

@@ -0,0 +1,7 @@
+from mage_ai.api.resources.BaseResource import BaseResource
+
+class GenericResource(BaseResource):
+    def __getattr__(self, name):
+        def _missing(*args, **kwargs):
+            return self.model.get(name)
+        return _missing()

mage_ai/api/views.py

@@ -102,7 +102,7 @@ def __determine_action(
      pk: Union[None, int, str] = None,
 ) -> Tuple[str, str]:
     if 'DELETE' == request.method:
-        return (DELETE, json.loads(request.body) if request.body else {})
+        return (DELETE, request.body_arguments)
 
     if 'GET' == request.method:
         if pk and child and child_pk:
@@ -115,10 +115,10 @@ def __determine_action(
             return (LIST, request.query_arguments)
 
     if 'POST' == request.method:
-        return (CREATE, json.loads(request.body) if request.body else {})
+        return (CREATE, request.body_arguments)
 
     if 'PUT' == request.method:
-        return (UPDATE, json.loads(request.body) if request.body else {})
+        return (UPDATE, request.body_arguments)
 
 
 def __meta(request) -> Dict:
@@ -135,12 +135,14 @@ def __meta_keys(request) -> List[str]:
 def __payload(request) -> Dict:
     if 'Content-Type' in request.headers and \
        'multipart/form-data' in request.headers.get('Content-Type'):
+
         parts = request.body.decode('utf-8', 'ignore').split('\r\n')
         idx = parts.index('Content-Disposition: form-data; name="json_root_body"')
         json_root_body = parts[idx + 2]
+
         return json.loads(json_root_body)
-    else:
-        return json.loads(request.body) if request.body else {}
+
+    return request.body_arguments
 
 
 def __query(request) -> Dict:

mage_ai/data_preparation/models/block/__init__.py

@@ -447,7 +447,7 @@ def create(
 
         if BlockType.DBT == block.type:
             if block.file_path and not block.file.exists():
-                os.makedirs(os.path.dirname(block.file_path), exist_ok=True)
+                block.file.create_parent_directories(block.file_path)
                 block.file.update_content('')
 
         self.after_create(

mage_ai/data_preparation/models/errors.py

@@ -0,0 +1,5 @@
+from mage_ai.errors.base import MageBaseException
+
+
+class FileExistsError(MageBaseException):
+    pass

mage_ai/data_preparation/models/file.py

@@ -1,3 +1,4 @@
+from mage_ai.data_preparation.models.errors import FileExistsError
 from mage_ai.data_preparation.repo_manager import get_repo_path
 from typing import Dict
 import aiofiles
@@ -32,11 +33,42 @@ def file_path(self) -> str:
         return os.path.join(self.repo_path, self.dir_path, self.filename)
 
     @classmethod
-    def create(self, filename, dir_path, repo_path=None):
+    def file_exists(self, file_path: str) -> bool:
+        return os.path.isfile(file_path)
+
+    @classmethod
+    def create_parent_directories(self, file_path: str) -> bool:
+        will_create = not self.file_exists(file_path)
+        if will_create:
+            os.makedirs(os.path.dirname(file_path), exist_ok=True)
+        return will_create
+
+    @classmethod
+    def create(
+        self,
+        filename,
+        dir_path,
+        content: str = None,
+        repo_path: str = None,
+        create_directories_if_not_exist: bool = True,
+        overwrite: bool = True,
+    ):
+
         repo_path = repo_path or get_repo_path()
         file = File(filename, dir_path, repo_path)
-        with open(file.file_path, 'w'):
-            pass
+        file_path = file.file_path
+
+        if self.file_exists(file_path) and not overwrite:
+            raise FileExistsError(f'File at {file_path} already exists.')
+
+        if create_directories_if_not_exist:
+            self.create_parent_directories(file_path)
+
+        write_type = 'wb' if content and type(content) is bytes else 'w'
+        with open(file_path, write_type) as f:
+            if content:
+                f.write(content)
+
         return file
 
     @classmethod
@@ -49,7 +81,7 @@ def get_all_files(self, repo_path):
         return traverse(os.path.basename(repo_path), True, repo_path)
 
     def exists(self) -> bool:
-        return os.path.isfile(self.file_path)
+        return self.file_exists(self.file_path)
 
     def content(self):
         try:

mage_ai/frontend/api/utils/fetcher.ts

@@ -10,7 +10,14 @@ type FetcherOptionsType = {
   body?: any;
   ctx?: any;
   method?: any;
-  onUploadProgress?: (progress: any) => void;
+  onUploadProgress?: (progress: any, opts?: {
+    body: {
+      [key: string]: number | string;
+    };
+    query: {
+      [key: string]: number | string;
+    };
+  }) => void;
   query?: any;
   token?: string;
 };
@@ -46,12 +53,13 @@ function preprocess(url: string, opts: FetcherOptionsType = {}) {
       } = file;
       const formData = new FormData();
       const key: string = Object.keys(body).filter(k => k !== 'file')[0];
+      const jsonRootBody = JSON.stringify({
+        api_key: API_KEY,
+        [key]: body[key],
+      });
       formData.set(
         'json_root_body',
-        JSON.stringify({
-          api_key: API_KEY,
-          [key]: body[key],
-        }),
+        jsonRootBody,
       );
       formData.append('file', file);
       data.body = formData;
@@ -118,7 +126,12 @@ export function buildFetchV2(urlArg: string, opts: FetcherOptionsType = {}) {
     data: data.body,
     headers,
     method,
-    onUploadProgress: opts?.onUploadProgress,
+    onUploadProgress: opts?.onUploadProgress
+      ? e => opts.onUploadProgress(e, {
+        body: opts?.body,
+        query: opts?.query,
+      })
+      : null,
     url: finalUrl,
   });
 }