Sync remote resources in groups edit

Dockerfile

@@ -4,6 +4,7 @@ MAINTAINER Francis Charette-Migneault
 RUN apt-get update && apt-get install -y \
 	build-essential \
 	supervisor \
+	cron \
 	curl \
 	libssl-dev \
 	libffi-dev \
@@ -23,4 +24,11 @@ COPY ./ $MAGPIE_DIR
 RUN make install -f $MAGPIE_DIR/Makefile
 RUN make docs -f $MAGPIE_DIR/Makefile
 
-CMD ["make", "start"]
+# magpie cron service
+ADD magpie-cron /etc/cron.d/magpie-cron
+RUN chmod 0644 /etc/cron.d/magpie-cron
+RUN touch ~/magpie_cron_status.log
+# set /etc/environment so that cron runs using the environment variables set by docker
+RUN env >> /etc/environment
+
+CMD make start

Makefile

@@ -143,7 +143,12 @@ install: sysinstall
 	@echo "Installing Magpie..."
 	python setup.py install
 
+.PHONY: cron
+cron:
+	@echo "Starting Cron service..."
+	cron
+
 .PHONY: start
-start: install
+start: cron install
 	@echo "Starting Magpie..."
 	exec gunicorn -b 0.0.0.0:2001 --paste "$(CUR_DIR)/magpie/magpie.ini" --workers 10 --preload

docs/.gitignore

@@ -0,0 +1,2 @@
+magpie*.rst
+modules.rst

env/magpie.env.example

@@ -6,6 +6,7 @@ MAGPIE_ADMIN_USER=admin
 MAGPIE_ADMIN_PASSWORD=qwerty
 MAGPIE_ANONYMOUS_USER=anonymous
 MAGPIE_USERS_GROUP=users
+MAGPIE_CRON_LOG=~/magpie_cron.log
 PHOENIX_USER=phoenix
 PHOENIX_PASSWORD=qwerty
 PHOENIX_PORT=8443

magpie-cron

@@ -0,0 +1 @@
+0 * * * * root /bin/bash -c "set -a ; . $MAGPIE_DIR/env/magpie.env ; . $MAGPIE_DIR/magpie/env/magpie.env ; . $MAGPIE_DIR/env/postgres.env ; . $MAGPIE_DIR/magpie/env/postgres.env ; set +a ; python -c 'from magpie.helpers.sync_resources import main; main()'" > ~/magpie_cron_status.log 2>&1

magpie/alembic/utils.py

@@ -0,0 +1,9 @@
+from magpie.definitions.sqlalchemy_definitions import *
+
+
+def has_column(context, table_name, column_name):
+    inspector = reflection.Inspector.from_engine(context.connection.engine)
+    for column in inspector.get_columns(table_name=table_name):
+        if column_name in column['name']:
+            return True
+    return False

magpie/alembic/versions/73639c63c4fc_unified_api_service.py

@@ -0,0 +1,85 @@
+"""unified api service
+
+Revision ID: 73639c63c4fc
+Revises: d01af1f2e445
+Create Date: 2018-09-27 16:12:02.282830
+
+"""
+import os, sys
+
+cur_file = os.path.abspath(__file__)
+root_dir = os.path.dirname(cur_file)  # version
+root_dir = os.path.dirname(root_dir)  # alembic
+root_dir = os.path.dirname(root_dir)  # magpie
+root_dir = os.path.dirname(root_dir)  # root
+sys.path.insert(0, root_dir)
+
+from alembic import op
+from alembic.context import get_context
+from magpie.definitions.sqlalchemy_definitions import *
+# from magpie.models import Service
+from sqlalchemy.sql import table
+from sqlalchemy import func
+
+Session = sessionmaker()
+
+# revision identifiers, used by Alembic.
+revision = '73639c63c4fc'
+down_revision = 'd01af1f2e445'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    context = get_context()
+    if isinstance(context.connection.engine.dialect, PGDialect):
+        # add 'sync_type' column if missing
+        op.add_column('services', sa.Column('sync_type', sa.UnicodeText(), nullable=True))
+
+        services = table('services',
+                         sa.Column('url', sa.UnicodeText()),
+                         sa.Column('type', sa.UnicodeText()),
+                         sa.Column('sync_type', sa.UnicodeText()),
+                         )
+
+        # transfer 'api' service types
+        op.execute(services.
+                   update().
+                   where(services.c.type == op.inline_literal('project-api')).
+                   values({'type': op.inline_literal('api'),
+                           'url': services.c.url + "/api",
+                           'sync_type': op.inline_literal('project-api')
+                           })
+                   )
+        op.execute(services.
+                   update().
+                   where(services.c.type == op.inline_literal('geoserver-api')).
+                   values({'type': op.inline_literal('api'),
+                           'sync_type': op.inline_literal('geoserver-api')
+                           })
+                   )
+
+
+def downgrade():
+    service = table('services',
+                    sa.Column('url', sa.UnicodeText()),
+                    sa.Column('type', sa.UnicodeText()),
+                    sa.Column('sync_type', sa.UnicodeText()),
+                    )
+
+    # transfer 'api' service types
+    op.execute(service.
+               update().
+               where(service.c.sync_type == op.inline_literal('project-api')).
+               values({'type': op.inline_literal('project-api'),
+                       'url': func.replace(service.c.url, '/api', ''),
+                       })
+               )
+    op.execute(service.
+               update().
+               where(service.c.sync_type == op.inline_literal('geoserver-api')).
+               values({'type': op.inline_literal('geoserver-api'),
+                       })
+               )
+
+    op.drop_column('services', 'sync_type')

magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py

@@ -39,17 +39,7 @@ def upgrade():
     context.connection.engine.dialect.supports_sane_multi_rowcount = False
 
     if isinstance(context.connection.engine.dialect, PGDialect):
-
-        # check if column exists, add it otherwise
-        inspector = reflection.Inspector.from_engine(context.connection.engine)
-        has_root_service_column = False
-        for column in inspector.get_columns(table_name='resources'):
-            if 'root_service_id' in column['name']:
-                has_root_service_column = True
-                break
-
-        if not has_root_service_column:
-            op.add_column('resources', sa.Column('root_service_id', sa.Integer(), nullable=True))
+        op.add_column('resources', sa.Column('root_service_id', sa.Integer(), nullable=True))
 
         # add existing resource references to their root service, loop through reference tree chain
         all_resources = session.query(models.Resource)
@@ -61,4 +51,6 @@ def upgrade():
 
 
 def downgrade():
-    pass
+    context = get_context()
+    if isinstance(context.connection.engine.dialect, PGDialect):
+        op.drop_column('resources', 'root_service_id')

magpie/alembic/versions/c352a98d570e_project_api_route_resource.py

@@ -32,7 +32,7 @@ def change_project_api_resource_type(new_type_name):
     if isinstance(context.connection.engine.dialect, PGDialect):
         # obtain service 'project-api'
         session = Session(bind=op.get_bind())
-        project_api_svc = models.Service.by_service_name('project-api', db_session=session)
+        project_api_svc = session.query(models.Service.resource_id).filter_by(resource_name='project-api').first()
 
         # nothing to edit if it doesn't exist, otherwise change resource types to 'route'
         if project_api_svc:

magpie/alembic/versions/d01af1f2e445_create_remote_sync_tables.py

@@ -0,0 +1,64 @@
+"""create remote sync tables
+
+Revision ID: d01af1f2e445
+Revises: c352a98d570e
+Create Date: 2018-09-11 10:56:23.779143
+
+"""
+import os, sys
+
+cur_file = os.path.abspath(__file__)
+root_dir = os.path.dirname(cur_file)  # version
+root_dir = os.path.dirname(root_dir)  # alembic
+root_dir = os.path.dirname(root_dir)  # magpie
+root_dir = os.path.dirname(root_dir)  # root
+sys.path.insert(0, root_dir)
+
+from alembic import op
+from magpie.definitions.sqlalchemy_definitions import *
+
+Session = sessionmaker()
+
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'd01af1f2e445'
+down_revision = 'c352a98d570e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('remote_resources',
+                    sa.Column('resource_id', sa.Integer(), primary_key=True,
+                              nullable=False, autoincrement=True),
+                    sa.Column('service_id',
+                              sa.Integer(),
+                              sa.ForeignKey('services.resource_id', onupdate='CASCADE', ondelete='CASCADE'),
+                              index=True,
+                              nullable=False),
+                    sa.Column('parent_id',
+                              sa.Integer(),
+                              sa.ForeignKey('remote_resources.resource_id', onupdate='CASCADE', ondelete='SET NULL'),
+                              nullable=True),
+                    sa.Column('ordering', sa.Integer(), default=0, nullable=False),
+                    sa.Column('resource_name', sa.Unicode(100), nullable=False),
+                    sa.Column('resource_type', sa.Unicode(30), nullable=False),
+                    )
+    op.create_table('remote_resources_sync_info',
+                    sa.Column('id', sa.Integer(), primary_key=True, nullable=False, autoincrement=True),
+                    sa.Column('service_id',
+                              sa.Integer(),
+                              sa.ForeignKey('services.resource_id', onupdate='CASCADE', ondelete='CASCADE'),
+                              index=True,
+                              nullable=False),
+                    sa.Column('remote_resource_id',
+                              sa.Integer(),
+                              sa.ForeignKey('remote_resources.resource_id', onupdate='CASCADE', ondelete='CASCADE')),
+                    sa.Column('last_sync', sa.DateTime(), nullable=True),
+                    )
+
+
+def downgrade():
+    op.drop_table('remote_resources_sync_info')
+    op.drop_table('remote_resources')

magpie/api/api_rest_schemas.py

@@ -455,6 +455,11 @@ class ServiceBodySchema(colander.MappingSchema):
         description="Type of the service",
         example="thredds"
     )
+    service_sync_type = colander.SchemaNode(
+        colander.String(),
+        description="Type of resource synchronization implementation.",
+        example="thredds"
+    )
     public_url = colander.SchemaNode(
         colander.String(),
         description="Proxy URL available for public access with permissions",
@@ -857,6 +862,11 @@ class Services_POST_BodySchema(colander.MappingSchema):
         description="Type of the service to create",
         example="wps"
     )
+    service_sync_type = colander.SchemaNode(
+        colander.String(),
+        description="Type of the service to create",
+        example="wps"
+    )
     service_url = colander.SchemaNode(
         colander.String(),
         description="Private URL of the service to create",

magpie/api/management/service/service_formats.py

@@ -13,6 +13,7 @@ def fmt_svc(svc, perms):
             u'service_url': str(svc.url),
             u'service_name': str(svc.resource_name),
             u'service_type': str(svc.type),
+            u'service_sync_type': svc.sync_type,
             u'resource_id': svc.resource_id,
             u'permission_names': sorted(service_type_dict[svc.type].permission_names if perms is None else perms)
         }

magpie/constants.py

@@ -53,6 +53,7 @@
 MAGPIE_ANONYMOUS_EMAIL = '{}@mail.com'.format(MAGPIE_ANONYMOUS_USER)
 MAGPIE_ANONYMOUS_GROUP = MAGPIE_ANONYMOUS_USER
 MAGPIE_USERS_GROUP = os.getenv('MAGPIE_USERS_GROUP', 'users')
+MAGPIE_CRON_LOG = os.getenv('MAGPIE_CRON_LOG', '~/magpie-cron.log')
 PHOENIX_USER = os.getenv('PHOENIX_USER', 'phoenix')
 PHOENIX_PASSWORD = os.getenv('PHOENIX_PASSWORD', 'qwerty')
 PHOENIX_PORT = int(os.getenv('PHOENIX_PORT', 8443))

magpie/definitions/ziggurat_definitions.py

@@ -1,7 +1,7 @@
 # import required definitions
 from ziggurat_foundations import ziggurat_model_init
 from ziggurat_foundations.models import groupfinder
-from ziggurat_foundations.models.base import get_db_session
+from ziggurat_foundations.models.base import get_db_session, BaseModel
 from ziggurat_foundations.models.external_identity import ExternalIdentityMixin
 from ziggurat_foundations.models.group import GroupMixin
 from ziggurat_foundations.models.group_permission import GroupPermissionMixin