Merge #178

178: Update dependencies r=kderme a=kderme

The old deleteCascade of persistent is deprecated, so we use the new way

Co-authored-by: kderme <k.dermenz@gmail.com>
Co-authored-by: Jean-Baptiste Giraudeau <jb@giraudeau.info>
Co-authored-by: kderme <kostas.dermentzis@iohk.io>

cabal.project

@@ -1,4 +1,4 @@
-index-state: 2021-04-20T00:00:00Z
+index-state: 2021-07-02T00:00:00Z
 
 packages:
     ./smash
@@ -15,6 +15,9 @@ constraints:
 package smash
   ghc-options: -Wall -Werror -Wredundant-constraints -Wincomplete-uni-patterns -Wincomplete-record-updates -Wpartial-fields -Wunused-imports
 
+package postgresql-libpq
+  flags: +use-pkg-config
+
 ------------------------------------------------------------------------------
 
 -- Disable all tests by default
@@ -49,15 +52,15 @@ package ouroboros-consensus-cardano
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/Win32-network
-  tag: 5b3d08c454f425da5cf045fe7865950d7c806691
-  --sha256: 0npyihbaqlih9abdbaj451lm1h0kl5braczy8vn41s3ivbbnvpcw
+  tag: 3825d3abf75f83f406c1f7161883c438dac7277d
+  --sha256: 19wahfv726fa3mqajpqdqhnl9ica3xmf68i254q45iyjcpj1psqx
 
 
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/cardano-db-sync
-  tag: 7815b392bfe9825bd89966bf96c04437e02c2429
-  --sha256: 0q4gx121ls69hgypwv8ii1n7x8i5w75zmwa49j2q0yb82rbz6ddi
+  tag: 7ab7a9a2863cb1cdd344fccafef163b2a8372dc5
+  --sha256: 02k5p17cq14xb0lgvzp1qg6df50yyl40s4acjqpwapn3mnf5ja97
   subdir:
     cardano-sync
     cardano-db
@@ -66,27 +69,30 @@ source-repository-package
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/cardano-base
-  tag: b6a215c42a28dc8b71b42946fe30256a333d34af
-  --sha256: 1l9nqsg6kfkfadlbvyl4afdf8v408zf0fn6dkihqf56g2mnk4i04
+  tag: cb0f19c85e5bb5299839ad4ed66af6fa61322cc4
+  --sha256: 0dnkfqcvbifbk3m5pg8kyjqjy0zj1l4vd23p39n6ym4q0bnib1cq
   subdir:
+    base-deriving-via
     binary
     binary/test
     cardano-crypto-class
     cardano-crypto-praos
+    measures
+    orphans-deriving-via
     slotting
     strict-containers
 
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/cardano-crypto
-  tag: ce8f1934e4b6252084710975bd9bbc0a4648ece4
-  --sha256: 1v2laq04piyj511b2m77hxjh9l1yd6k9kc7g6bjala4w3zdwa4ni
+  tag: 07397f0e50da97eaa0575d93bee7ac4b2b2576ec
+  --sha256: 06sdx5ndn2g722jhpicmg96vsrys89fl81k8290b3lr6b1b0w4m3
 
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/cardano-ledger-specs
-  tag: ec9c77edbf5700a4b2ece8f97a1e313df06abc97
-  --sha256: 1ia8x9dnw36y0xazg7xg263ax9mamw9w4sg460cmibj3wv49im4w
+  tag: d5b184a820853c7ba202efd615b8fadca1acb52c
+  --sha256: 04k5p6qwmfdza65gl5319r1ahdfwjnyqgzpfxdx0x2g5jcbimar4
   subdir:
     alonzo/impl
     byron/crypto
@@ -107,8 +113,8 @@ source-repository-package
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/cardano-node
-  tag: 48429531f0d3d71fadce9a5971bf56a6df396f2d
-  --sha256: 0arhmmxb86ggfc73rbyz1fj43glj9z2f8w5416cix2wxkvlm8bys
+  tag: 9a6a6c81e3aebfaf757b562c823146c7da601e1c
+  --sha256: 1xiqrx3hf2s7j62clzzmlim81g7v2dvmirv78zf9gp9m1lqxzan6
   subdir:
     cardano-api
     cardano-config
@@ -144,11 +150,18 @@ source-repository-package
     plugins/scribe-systemd
     tracer-transformers
 
+-- Using a fork until our patches can be merged upstream
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/optparse-applicative
+  tag: 84bcc6f18992a441886589a117249bfface8630e
+  --sha256: 09pr7m9gjsk8889m9d3mj75p69viv1acnrq63dgb11jl5gnnsblr
+
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/ouroboros-network
-  tag: e338f2cf8e1078fbda9555dd2b169c6737ef6774
-  --sha256: 12x81hpjyw2cpkazfalz6bw2wgr6ax7bnmlxl2rlfakkvsjfgaqd
+  tag: d070bad7ce389a4b2ff7fb4fcb7937fdeca80f3a
+  --sha256: 0jzdwjgqcj06b0rvwyh61cgf23dlh62lcn8z7dbm7wxwjjgpkjb1
   subdir:
     cardano-client
     io-sim
@@ -168,8 +181,8 @@ source-repository-package
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/plutus
-  tag: 523f349f3d68db07c98150734793ed7003d1f562
-  --sha256: 0vp6wiv1fz5bzvw90pdwv96nck78m5s91xiwjhkksq06l1yqr3ps
+  tag: 8c83c4abe211b4bbcaca3cdf1b2c0e38d0eb683f
+  --sha256: 1643s1g3jlm9pgalpc3vpij1zqb1n8yv8irq6qc43gs9bvl0wc3l
   subdir:
     plutus-core
     plutus-ledger-api
@@ -184,7 +197,6 @@ allow-newer:
 -- Something in plutus-core requries this.
 source-repository-package
   type: git
-  location: https://github.com/Quid2/flat.git
-  tag: 95e5d7488451e43062ca84d5376b3adcc465f1cd
-  --sha256: 06l31x3y93rjpryvlxnpsyq2zyxvb0z6lik6yq2fvh36i5zwvwa3
-
+  location: https://github.com/michaelpj/flat.git
+  tag: ee59880f47ab835dbd73bea0847dab7869fc20d8
+  --sha256: 1lrzknw765pz2j97nvv9ip3l1mcpf2zr4n56hwlz0rk7wq7ls4cm

nix/default.nix

@@ -8,12 +8,7 @@ let
     // sourcesOverride;
   iohkNix = import sources.iohk-nix {};
   haskellNix = import sources."haskell.nix" {};
-  # use our own nixpkgs if it exists in our sources,
-  # otherwise use iohkNix default nixpkgs.
-  nixpkgs = if (sources ? nixpkgs)
-    then (builtins.trace "Not using IOHK default nixpkgs (use 'niv drop nixpkgs' to use default for better sharing)"
-      sources.nixpkgs)
-    else iohkNix.nixpkgs;
+  nixpkgs = haskellNix.sources.nixpkgs-2105;
 
   # for inclusion in pkgs:
   overlays =

nix/haskell.nix

@@ -83,6 +83,10 @@ let
           inherit preCheck;
         };
       }
+      {
+        packages.cardano-db-sync.package.extraSrcFiles = ["../schema/*.sql"];
+        packages.cardano-db-sync-extended.package.extraSrcFiles = ["../cardano-db-sync/Setup.hs" "../schema/*.sql"];
+      }
       # TODO: Compile all local packages with -Werror:
       #{
       #  packages = lib.genAttrs projectPackagesNames

nix/sources.json

@@ -17,10 +17,10 @@
         "homepage": "https://input-output-hk.github.io/haskell.nix",
         "owner": "input-output-hk",
         "repo": "haskell.nix",
-        "rev": "5bd075250123ee7a62aa6bbf2a4861dda6e8621d",
-        "sha256": "0c9y0lhj6z5ri38pww1cdh4yfh7dk7bi4ycwb7lfvb5agpp5nj73",
+        "rev": "8407c0fcdb7591d1a323f5ca39abc607f226d0dc",
+        "sha256": "0jg1kv4adayg4irza1mjgxbhawk6ss7nzs0yl2p7c1knf5zzgifa",
         "type": "tarball",
-        "url": "https://github.com/input-output-hk/haskell.nix/archive/5bd075250123ee7a62aa6bbf2a4861dda6e8621d.tar.gz",
+        "url": "https://github.com/input-output-hk/haskell.nix/archive/8407c0fcdb7591d1a323f5ca39abc607f226d0dc.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "iohk-nix": {

schema/migration-2-0009-20210818.sql

@@ -0,0 +1,37 @@
+-- Persistent generated migration.
+
+CREATE FUNCTION migrate() RETURNS void AS $$
+DECLARE
+  next_version int ;
+BEGIN
+  SELECT stage_two + 1 INTO next_version FROM schema_version ;
+  IF next_version = 9 THEN
+    ALTER TABLE "schema_version" ALTER COLUMN "id" TYPE INT8;
+    ALTER TABLE "pool_metadata_reference" ALTER COLUMN "pool_id" TYPE text;
+    ALTER TABLE "pool_metadata_reference" ALTER COLUMN "url" TYPE text;
+    ALTER TABLE "pool_metadata_reference" ALTER COLUMN "hash" TYPE text;
+    ALTER TABLE "pool_metadata" ALTER COLUMN "pool_id" TYPE text;
+    ALTER TABLE "pool_metadata" ALTER COLUMN "ticker_name" TYPE text;
+    ALTER TABLE "pool_metadata" ALTER COLUMN "hash" TYPE text;
+    ALTER TABLE "pool_metadata" ALTER COLUMN "metadata" TYPE text;
+    ALTER TABLE "pool_metadata" DROP CONSTRAINT "pool_metadata_pmr_id_fkey";
+    ALTER TABLE "pool_metadata" ADD CONSTRAINT "pool_metadata_pmr_id_fkey" FOREIGN KEY("pmr_id") REFERENCES "pool_metadata_reference"("id") ON DELETE CASCADE  ON UPDATE RESTRICT;
+    ALTER TABLE "pool" ALTER COLUMN "pool_id" TYPE text;
+    ALTER TABLE "retired_pool" ALTER COLUMN "pool_id" TYPE text;
+    ALTER TABLE "pool_metadata_fetch_error" ALTER COLUMN "pool_id" TYPE text;
+    ALTER TABLE "pool_metadata_fetch_error" ALTER COLUMN "pool_hash" TYPE text;
+    ALTER TABLE "pool_metadata_fetch_error" DROP CONSTRAINT "pool_metadata_fetch_error_pmr_id_fkey";
+    ALTER TABLE "pool_metadata_fetch_error" ADD CONSTRAINT "pool_metadata_fetch_error_pmr_id_fkey" FOREIGN KEY("pmr_id") REFERENCES "pool_metadata_reference"("id") ON DELETE CASCADE  ON UPDATE RESTRICT;
+    ALTER TABLE "delisted_pool" ALTER COLUMN "pool_id" TYPE text;
+    ALTER TABLE "reserved_ticker" ALTER COLUMN "name" TYPE text;
+    ALTER TABLE "reserved_ticker" ALTER COLUMN "pool_hash" TYPE text;
+    -- Hand written SQL statements can be added here.
+    UPDATE schema_version SET stage_two = 9 ;
+    RAISE NOTICE 'DB has been migrated to stage_two version %', next_version ;
+  END IF ;
+END ;
+$$ LANGUAGE plpgsql ;
+
+SELECT migrate() ;
+
+DROP FUNCTION migrate() ;

shell.nix

@@ -26,7 +26,7 @@ let
       pkgconfig
       sqlite-interactive
       tmux
-      git
+      pkgs.git
     ];
 
     # Prevents cabal from choosing alternate plans, so that

smash/src/Cardano/SMASH/DBSync/Db/Delete.hs

@@ -10,7 +10,7 @@ module Cardano.SMASH.DBSync.Db.Delete
 
 import           Cardano.Prelude                hiding (Meta)
 
-import           Database.Persist.Sql           (SqlBackend, deleteCascade,
+import           Database.Persist.Sql           (SqlBackend, delete,
                                                  selectKeysList, (==.))
 
 import           Cardano.SMASH.DBSync.Db.Schema
@@ -21,25 +21,25 @@ import qualified Cardano.SMASH.DBSync.Db.Types  as Types
 deleteDelistedPool :: MonadIO m => Types.PoolId -> ReaderT SqlBackend m Bool
 deleteDelistedPool poolId = do
   keys <- selectKeysList [ DelistedPoolPoolId ==. poolId ] []
-  mapM_ deleteCascade keys
+  mapM_ delete keys
   pure $ not (null keys)
 
 -- | Delete a retired pool if it exists. Returns 'True' if it did exist and has been
 -- deleted and 'False' if it did not exist.
 deleteRetiredPool :: MonadIO m => Types.PoolId -> ReaderT SqlBackend m Bool
 deleteRetiredPool poolId = do
   keys <- selectKeysList [ RetiredPoolPoolId ==. poolId ] []
-  mapM_ deleteCascade keys
+  mapM_ delete keys
   pure $ not (null keys)
 
 deleteAdminUser :: MonadIO m => AdminUser -> ReaderT SqlBackend m Bool
 deleteAdminUser adminUser = do
   keys <- selectKeysList [ AdminUserUsername ==. adminUserUsername adminUser, AdminUserPassword ==. adminUserPassword adminUser ] []
-  mapM_ deleteCascade keys
+  mapM_ delete keys
   pure $ not (null keys)
 
 deleteCascadeSlotNo :: MonadIO m => Word64 -> ReaderT SqlBackend m Bool
 deleteCascadeSlotNo slotNo = do
   keys <- selectKeysList [ BlockSlotNo ==. Just slotNo ] []
-  mapM_ deleteCascade keys
+  mapM_ delete keys
   pure $ not (null keys)

smash/src/Cardano/SMASH/DBSync/Db/Query.hs

@@ -39,7 +39,7 @@ import           Control.Monad.Extra            (mapMaybeM)
 
 import           Data.Time.Clock                (UTCTime)
 
-import           Database.Esqueleto             (Entity, PersistField, SqlExpr,
+import           Database.Esqueleto.Legacy      (Entity, PersistField, SqlExpr,
                                                  Value, ValueList, countRows,
                                                  desc, entityVal, from,
                                                  isNothing, just, limit, notIn,

smash/src/Cardano/SMASH/DBSync/Db/Run.hs

@@ -41,7 +41,7 @@ import           Database.Persist.Sql                 (IsolationLevel (..),
                                                        runSqlConnWithIsolation)
 import           Database.PostgreSQL.Simple           (connectPostgreSQL)
 
-import           Database.Esqueleto
+import           Database.Esqueleto.Legacy
 import           Database.Esqueleto.Internal.Internal (Mode (..), SqlSelect,
                                                        initialIdentState,
                                                        toRawSql)

smash/src/Cardano/SMASH/DBSync/Db/Schema.hs

@@ -38,7 +38,6 @@ import qualified Cardano.SMASH.DBSync.Db.Types as Types
 
 share
   [ mkPersist sqlSettings
-  , mkDeleteCascade sqlSettings
   , mkMigrate "migrateCardanoDb"
   ]
   [persistLowerCase|
@@ -69,7 +68,7 @@ share
     tickerName          Types.TickerName          sqltype=text
     hash                Types.PoolMetadataHash    sqltype=text
     metadata            Types.PoolMetadataRaw     sqltype=text
-    pmrId               PoolMetadataReferenceId Maybe
+    pmrId               PoolMetadataReferenceId Maybe OnDeleteCascade
     UniquePoolMetadata  poolId hash
 
   -- The pools themselves (identified by the owner vkey hash)
@@ -92,7 +91,7 @@ share
     fetchTime           UTCTime                   sqltype=timestamp
     poolId              Types.PoolId              sqltype=text
     poolHash            Types.PoolMetadataHash    sqltype=text
-    pmrId               PoolMetadataReferenceId
+    pmrId               PoolMetadataReferenceId   OnDeleteCascade
     fetchError          Text
     retryCount          Word                      sqltype=uinteger
     UniquePoolMetadataFetchError fetchTime poolId poolHash retryCount

smash/src/Cardano/SMASH/DBSyncPlugin.hs

@@ -45,8 +45,10 @@ import qualified Cardano.SMASH.DBSync.Db.Query      as DB
 import           Cardano.Sync.Error
 import           Cardano.Sync.Types                 as DbSync
 
-import           Cardano.Sync.LedgerState           (applyBlock,
-                                                     saveLedgerStateMaybe)
+import           Cardano.Sync.LedgerState           (LedgerStateSnapshot (..),
+                                                     applyBlock,
+                                                     getAlonzoPParams,
+                                                     saveCleanupState)
 
 import           Cardano.Sync                       (SyncEnv (..),
                                                      SyncNodePlugin (..))
@@ -132,27 +134,30 @@ insertDefaultBlock
     -> ReaderT SqlBackend (LoggingT IO) (Either SyncNodeError ())
 insertDefaultBlock dataLayer tracer env (BlockDetails cblk details) = do
 
-  -- Calculate the new ledger state to pass to the DB insert functions but do not yet
-  -- update ledgerStateVar.
-  lStateSnap <- liftIO $ applyBlock (envLedger env) cblk details
-
-  res <- case cblk of
-            BlockByron blk -> do
-              insertByronBlock tracer blk details
-            BlockShelley blk -> do
-              insertShelleyBlock Shelley dataLayer tracer env (Generic.fromShelleyBlock blk) details
-            BlockAllegra blk -> do
-              insertShelleyBlock Allegra dataLayer tracer env (Generic.fromAllegraBlock blk) details
-            BlockMary blk -> do
-              insertShelleyBlock Mary dataLayer tracer env (Generic.fromMaryBlock blk) details
-            BlockAlonzo blk ->
-              insertShelleyBlock Alonzo dataLayer tracer env (Generic.fromAlonzoBlock blk) details
-
-  -- Now we update it in ledgerStateVar and (possibly) store it to disk.
-  liftIO $ saveLedgerStateMaybe (envLedger env)
-                lStateSnap (isSyncedWithinSeconds details 60)
+    -- Calculate the new ledger state to pass to the DB insert functions but do not yet
+    -- update ledgerStateVar.
+    lStateSnap <- liftIO $ applyBlock (envLedger env) cblk details
+    mkSnapshotMaybe lStateSnap (isSyncedWithinSeconds details 60)
+    res <- case cblk of
+              BlockByron blk -> do
+                insertByronBlock tracer blk details
+              BlockShelley blk -> do
+                insertShelleyBlock Shelley dataLayer tracer env (Generic.fromShelleyBlock blk) details
+              BlockAllegra blk -> do
+                insertShelleyBlock Allegra dataLayer tracer env (Generic.fromAllegraBlock blk) details
+              BlockMary blk -> do
+                insertShelleyBlock Mary dataLayer tracer env (Generic.fromMaryBlock blk) details
+              BlockAlonzo blk -> do
+                let pp = getAlonzoPParams $ lssState lStateSnap
+                insertShelleyBlock Alonzo dataLayer tracer env (Generic.fromAlonzoBlock pp blk) details
+
+    pure res
+  where
 
-  pure res
+    mkSnapshotMaybe snapshot syncState =
+      whenJust (lssNewEpoch snapshot) $ \newEpoch -> do
+        let newEpochNo = Generic.neEpoch newEpoch
+        liftIO $ saveCleanupState (envLedger env) (lssOldState snapshot) syncState  (Just $ newEpochNo - 1)
 
 
 -- We don't care about Byron, no pools there
@@ -251,7 +256,7 @@ insertCertificate
     -> SyncEnv
     -> Generic.TxCertificate
     -> ExceptT SyncNodeError m ()
-insertCertificate dataLayer blockNumber tracer _env (Generic.TxCertificate _idx cert) =
+insertCertificate dataLayer blockNumber tracer _env (Generic.TxCertificate _ _idx cert) =
   case cert of
     Shelley.DCertDeleg _deleg ->
         -- Since at some point we start to have a large number of delegation

smash/src/Cardano/SMASH/DBSyncRun.hs

@@ -34,7 +34,7 @@ import           Cardano.Sync.Database             (runDbThread)
 
 import           Control.Monad.Trans.Maybe
 
-import           Database.Esqueleto
+import           Database.Esqueleto.Legacy
 
 import qualified Cardano.BM.Setup                  as Logging
 import           Cardano.BM.Trace                  (Trace, logInfo, modifyName)

smash/src/Cardano/SMASH/Offline.hs

@@ -51,7 +51,7 @@ import qualified Cardano.SMASH.DBSync.Db.Schema   as DB
 
 import qualified Data.ByteString.Base16           as B16
 
-import           Database.Esqueleto               (Entity (..), InnerJoin (..),
+import           Database.Esqueleto.Legacy        (Entity (..), InnerJoin (..),
                                                    SqlExpr, Value, ValueList,
                                                    desc, from, groupBy, in_,
                                                    just, max_, notExists, on,

smash/test/MigrationSpec.hs

@@ -91,7 +91,7 @@ migrationTest = do
 
     -- TODO(KS): This version HAS to be changed manually so we don't mess up the
     -- migration.
-    let expected = SchemaVersion 1 8 0
+    let expected = SchemaVersion 1 9 0
     actual <- getDbSchemaVersion
     unless (expected == actual) $
         panic $ mconcat