/db/table/-/upsert API

Close #1878

Also made a few tweaks to how _r works in tokens and actors,
refs #1855 - I needed that mechanism for the tests.

datasette/app.py

@@ -40,7 +40,7 @@
     PermissionsDebugView,
     MessagesDebugView,
 )
-from .views.table import TableView, TableInsertView, TableDropView
+from .views.table import TableView, TableInsertView, TableUpsertView, TableDropView
 from .views.row import RowView, RowDeleteView, RowUpdateView
 from .renderer import json_renderer
 from .url_builder import Urls
@@ -1292,6 +1292,10 @@ def add_route(view, regex):
             TableInsertView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/insert$",
         )
+        add_route(
+            TableUpsertView.as_view(self),
+            r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/upsert$",
+        )
         add_route(
             TableDropView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/drop$",

datasette/default_permissions.py

@@ -85,7 +85,7 @@ def permission_allowed_actor_restrictions(actor, action, resource):
             if action_initials in database_allowed:
                 return None
     # Or the current table? That's any time the resource is (database, table)
-    if not isinstance(resource, str) and len(resource) == 2:
+    if resource is not None and not isinstance(resource, str) and len(resource) == 2:
         database, table = resource
         table_allowed = _r.get("t", {}).get(database, {}).get(table)
         # TODO: What should this do for canned queries?
@@ -138,6 +138,8 @@ def actor_from_request(datasette, request):
             # Expired
             return None
     actor = {"id": decoded["a"], "token": "dstok"}
+    if "_r" in decoded:
+        actor["_r"] = decoded["_r"]
     if duration:
         actor["token_expires"] = created + duration
     return actor

datasette/views/special.py

@@ -316,21 +316,37 @@ async def example_links(self, request):
                     request.actor, "insert-row", (name, table)
                 ):
                     pks = await db.primary_keys(table)
-                    table_links.append(
-                        {
-                            "path": self.ds.urls.table(name, table) + "/-/insert",
-                            "method": "POST",
-                            "label": "Insert rows into {}".format(table),
-                            "json": {
-                                "rows": [
-                                    {
-                                        column: None
-                                        for column in await db.table_columns(table)
-                                        if column not in pks
-                                    }
-                                ]
+                    table_links.extend(
+                        [
+                            {
+                                "path": self.ds.urls.table(name, table) + "/-/insert",
+                                "method": "POST",
+                                "label": "Insert rows into {}".format(table),
+                                "json": {
+                                    "rows": [
+                                        {
+                                            column: None
+                                            for column in await db.table_columns(table)
+                                            if column not in pks
+                                        }
+                                    ]
+                                },
                             },
-                        }
+                            {
+                                "path": self.ds.urls.table(name, table) + "/-/upsert",
+                                "method": "POST",
+                                "label": "Upsert rows into {}".format(table),
+                                "json": {
+                                    "rows": [
+                                        {
+                                            column: None
+                                            for column in await db.table_columns(table)
+                                            if column not in pks
+                                        }
+                                    ]
+                                },
+                            },
+                        ]
                     )
                 if await self.ds.permission_allowed(
                     request.actor, "drop-table", (name, table)

datasette/views/table.py

@@ -1074,9 +1074,18 @@ class TableInsertView(BaseView):
     def __init__(self, datasette):
         self.ds = datasette
 
-    async def _validate_data(self, request, db, table_name):
+    async def _validate_data(self, request, db, table_name, pks, upsert):
         errors = []
 
+        pks_list = []
+        if isinstance(pks, str):
+            pks_list = [pks]
+        else:
+            pks_list = list(pks)
+
+        if not pks_list:
+            pks_list = ["rowid"]
+
         def _errors(errors):
             return None, errors, {}
 
@@ -1134,7 +1143,18 @@ def _errors(errors):
 
         # Validate columns of each row
         columns = set(await db.table_columns(table_name))
+        columns.update(pks_list)
+
         for i, row in enumerate(rows):
+            if upsert:
+                # It MUST have the primary key
+                missing_pks = [pk for pk in pks_list if pk not in row]
+                if missing_pks:
+                    errors.append(
+                        'Row {} is missing primary key column(s): "{}"'.format(
+                            i, '", "'.join(missing_pks)
+                        )
+                    )
             invalid_columns = set(row.keys()) - columns
             if invalid_columns:
                 errors.append(
@@ -1146,7 +1166,7 @@ def _errors(errors):
             return _errors(errors)
         return rows, errors, extras
 
-    async def post(self, request):
+    async def post(self, request, upsert=False):
         try:
             resolved = await self.ds.resolve_table(request)
         except NotFound as e:
@@ -1159,45 +1179,106 @@ async def post(self, request):
         db = self.ds.get_database(database_name)
         if not await db.table_exists(table_name):
             return _error(["Table not found: {}".format(table_name)], 404)
-        # Must have insert-row permission
-        if not await self.ds.permission_allowed(
-            request.actor, "insert-row", resource=(database_name, table_name)
-        ):
-            return _error(["Permission denied"], 403)
-        rows, errors, extras = await self._validate_data(request, db, table_name)
+
+        if upsert:
+            # Must have insert-row AND upsert-row permissions
+            if not (
+                await self.ds.permission_allowed(
+                    request.actor, "insert-row", database_name, table_name
+                )
+                and await self.ds.permission_allowed(
+                    request.actor, "update-row", database_name, table_name
+                )
+            ):
+                return _error(
+                    ["Permission denied: need both insert-row and update-row"], 403
+                )
+        else:
+            # Must have insert-row permission
+            if not await self.ds.permission_allowed(
+                request.actor, "insert-row", resource=(database_name, table_name)
+            ):
+                return _error(["Permission denied"], 403)
+
+        if not db.is_mutable:
+            return _error(["Database is immutable"], 403)
+
+        pks = await db.primary_keys(table_name)
+
+        rows, errors, extras = await self._validate_data(
+            request, db, table_name, pks, upsert
+        )
         if errors:
             return _error(errors, 400)
 
+        # No that we've passed pks to _validate_data it's safe to
+        # fix the rowids case:
+        if not pks:
+            pks = ["rowid"]
+
         ignore = extras.get("ignore")
         replace = extras.get("replace")
 
+        if upsert and (ignore or replace):
+            return _error(["Upsert does not support ignore or replace"], 400)
+
         should_return = bool(extras.get("return", False))
-        # Insert rows
-        def insert_rows(conn):
+        row_pk_values_for_later = []
+        if should_return and upsert:
+            row_pk_values_for_later = [tuple(row[pk] for pk in pks) for row in rows]
+
+        def insert_or_upsert_rows(conn):
             table = sqlite_utils.Database(conn)[table_name]
-            if should_return:
+            kwargs = {}
+            if upsert:
+                kwargs["pk"] = pks[0] if len(pks) == 1 else pks
+            else:
+                kwargs = {"ignore": ignore, "replace": replace}
+            if should_return and not upsert:
                 rowids = []
+                method = table.upsert if upsert else table.insert
                 for row in rows:
-                    rowids.append(
-                        table.insert(row, ignore=ignore, replace=replace).last_rowid
-                    )
+                    rowids.append(method(row, **kwargs).last_rowid)
                 return list(
                     table.rows_where(
                         "rowid in ({})".format(",".join("?" for _ in rowids)),
                         rowids,
                     )
                 )
             else:
-                table.insert_all(rows, ignore=ignore, replace=replace)
+                method_all = table.upsert_all if upsert else table.insert_all
+                method_all(rows, **kwargs)
 
         try:
-            rows = await db.execute_write_fn(insert_rows)
+            rows = await db.execute_write_fn(insert_or_upsert_rows)
         except Exception as e:
             return _error([str(e)])
         result = {"ok": True}
         if should_return:
-            result["rows"] = rows
-        return Response.json(result, status=201)
+            if upsert:
+                # Fetch based on initial input IDs
+                where_clause = " OR ".join(
+                    ["({})".format(" AND ".join("{} = ?".format(pk) for pk in pks))]
+                    * len(row_pk_values_for_later)
+                )
+                args = list(itertools.chain.from_iterable(row_pk_values_for_later))
+                fetched_rows = await db.execute(
+                    "select {}* from [{}] where {}".format(
+                        "rowid, " if pks == ["rowid"] else "", table_name, where_clause
+                    ),
+                    args,
+                )
+                result["rows"] = [dict(r) for r in fetched_rows.rows]
+            else:
+                result["rows"] = rows
+        return Response.json(result, status=200 if upsert else 201)
+
+
+class TableUpsertView(TableInsertView):
+    name = "table-upsert"
+
+    async def post(self, request):
+        return await super().post(request, upsert=True)
 
 
 class TableDropView(BaseView):

docs/json_api.rst

@@ -527,7 +527,7 @@ To insert multiple rows at a time, use the same API method but send a list of di
         ]
     }
 
-If successful, this will return a ``201`` status code and an empty ``{}`` response body.
+If successful, this will return a ``201`` status code and a ``{"ok": true}`` response body.
 
 To return the newly inserted rows, add the ``"return": true`` key to the request body:
 
@@ -582,6 +582,116 @@ Pass ``"ignore": true`` to ignore these errors and insert the other rows:
 
 Or you can pass ``"replace": true`` to replace any rows with conflicting primary keys with the new values.
 
+.. _TableUpsertView:
+
+Upserting rows
+~~~~~~~~~~~~~~
+
+An upsert is an insert or update operation. If a row with a matching primary key already exists it will be updated - otherwise a new row will be inserted.
+
+The upsert API is mostly the same shape as the :ref:`insert API <TableInsertView>`. It requires both the :ref:`permissions_insert_row` and :ref:`permissions_update_row` permissions.
+
+::
+
+    POST /<database>/<table>/-/upsert
+    Content-Type: application/json
+    Authorization: Bearer dstok_<rest-of-token>
+
+.. code-block:: json
+
+    {
+        "rows": [
+            {
+                "id": 1,
+                "title": "Updated title for 1",
+                "description": "Updated description for 1"
+            },
+            {
+                "id": 2,
+                "description": "Updated description for 2",
+            },
+            {
+                "id": 3,
+                "title": "Item 3",
+                "description": "Description for 3"
+            }
+        ]
+    }
+
+Imagine a table with a primary key of ``id`` and which already has rows with ``id`` values of ``1`` and ``2``.
+
+The above example will:
+
+- Update the row with ``id`` of ``1`` to set both ``title`` and ``description`` to the new values
+- Update the row with ``id`` of ``2`` to set ``title`` to the new value - ``description`` will be left unchanged
+- Insert a new row with ``id`` of ``3`` and both ``title`` and ``description`` set to the new values
+
+Similar to ``/-/insert``, a ``row`` key with an object can be used instead of a ``rows`` array to upsert a single row.
+
+If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body.
+
+Add ``"return": true`` to the request body to return full copies of the affected rows after they have been inserted or updated:
+
+.. code-block:: json
+
+    {
+        "rows": [
+            {
+                "id": 1,
+                "title": "Updated title for 1",
+                "description": "Updated description for 1"
+            },
+            {
+                "id": 2,
+                "description": "Updated description for 2",
+            },
+            {
+                "id": 3,
+                "title": "Item 3",
+                "description": "Description for 3"
+            }
+        ],
+        "return": true
+    }
+
+This will return the following:
+
+.. code-block:: json
+
+    {
+        "ok": true,
+        "rows": [
+            {
+                "id": 1,
+                "title": "Updated title for 1",
+                "description": "Updated description for 1"
+            },
+            {
+                "id": 2,
+                "title": "Item 2",
+                "description": "Updated description for 2"
+            },
+            {
+                "id": 3,
+                "title": "Item 3",
+                "description": "Description for 3"
+            }
+        ]
+    }
+
+When using upsert you must provide the primary key column (or columns if the table has a compound primary key) for every row, or you will get a ``400`` error:
+
+.. code-block:: json
+
+    {
+        "ok": false,
+        "errors": [
+            "Row 0 is missing primary key column(s): \"id\""
+        ]
+    }
+
+If your table does not have an explicit primary key you should pass the SQLite ``rowid`` key instead.
+
 .. _RowUpdateView:
 
 Updating a row