Dyalog · rikedyp · Oct 22, 2025 · Oct 16, 2025 · Oct 16, 2025 · Oct 16, 2025
diff --git a/.gitignore b/.gitignore
@@ -8,3 +8,9 @@ dcms_db/
 env
 DCMS.code-workspace
 CI/run-tests-in-docker.sh
+packages/*
+!packages/apl-dependencies.txt
+!packages/apl-buildlist.json
+nuget-packages/*
+!nuget-packages/_nuget-packages.csproj
+dcms.dws
diff --git a/APLSource/RefreshData.aplf b/APLSource/RefreshData.aplf
@@ -1,8 +1,8 @@
  r←RefreshData timer
- :Trap 202
+ :Trap 202 404
      import.youtube.RefreshData''
  :Else
-     ⎕←'DATA NOT REFRESHED. Could not access YouTube API.'
+     ⎕←'DATA NOT REFRESHED. ',⎕DMX.Message
  :EndTrap
  read.BuildCache''
  ⎕←r←'Refreshed data at ',⍕⎕TS
diff --git a/APLSource/Run.aplf b/APLSource/Run.aplf
@@ -1,19 +1,5 @@
  Run debug;app_dir;LoadDependency
- ⎕←'Setting up...'
- ⎕←'Loading dependencies...'
- :If 0=#.⎕NC'Conga'
-     'Conga'#.⎕CY'conga'
-     #.DRC←#.Conga.Init''
- :EndIf
- 'GLOBAL'⎕NS''
  GLOBAL.debug←debug
- 'SQA'SQL.⎕CY'sqapl'
- #.⎕CY'isolate'
- LoadDependency←{0=##.⎕NC ⍵:⎕SE.Link.Import ##,⍥⊂GetEnv ⍵}
- LoadDependency¨'Jarvis' 'HttpCommand'
- ⎕←'Setting app_dir'
- GLOBAL.app_dir←app_dir←{'/'=⊃⌽⍵:⍵ ⋄ '/',⍨⍵}GetEnv'app_dir'
- Unidecode.MakeCharMap app_dir,'/APLSource/charmap.json'
 ⍝ Initialise SQL schema
  GLOBAL.secrets←ReadJSON GetEnv'SECRETS'
  ⎕←'Initialising SQAPL...'

diff --git a/APLSource/Setup.aplf b/APLSource/Setup.aplf
@@ -0,0 +1,27 @@
+ Setup
+⍝ Load dependencies into the active workspace
+ ⎕←'Setting up...'
+ :Trap 0
+     'GLOBAL'⎕NS''
+     GLOBAL.app_dir←app_dir←{'/'=⊃⌽⍵:⍵ ⋄ '/',⍨⍵}GetEnv'APP_DIR'
+     ⎕←'App directory is ',app_dir
+     ⎕←'Loading dependencies...'
+     :If 0=#.⎕NC'Conga'
+         'Conga'#.⎕CY'conga'
+         #.DRC←#.Conga.Init''
+         ⎕←'Conga initalised'
+     :EndIf
+     'SQA'SQL.⎕CY'sqapl'
+     ⎕←'SQAPL loaded into SQL.SQA'
+     #.⎕CY'isolate'
+     ⎕←'Isolate loaded into #'
+     Unidecode.MakeCharMap app_dir,'/APLSource/charmap.json'
+     ⎕←'Loading Tatin packages...'
+     ⎕←⍪⎕SE.Tatin.LoadDependencies (GetEnv'PKG') #
+     ⎕←'Loading NuGet packages...'
+     ⎕←⍪⎕USING←##.NuGet.Using GetEnv'PKG_NUGET'
+ :Else
+     ⎕←'Could not set up DCMS'
+     ⎕←1⎕JSON⎕OPT'Compact'0⊢⎕DMX
+     ⎕OFF 1
+ :EndTrap
diff --git a/APLSource/Version.aplf b/APLSource/Version.aplf
@@ -1,2 +1,2 @@
  version←Version
- version←'3.23.0'
+ version←'3.24.0'
diff --git a/APLSource/import/youtube/GetVideos.aplf b/APLSource/import/youtube/GetVideos.aplf
@@ -7,11 +7,12 @@
      q,←'&part=snippet'
      q,←'&maxResults=100'
      r←#.HttpCommand.GetJSON'Get'q
-     0≠r.rc:r.msg ⎕SIGNAL 202
+     0≠r.rc:⎕SIGNAL⊂('EN'202)('Message' r.msg)('Vendor' (#.DCMS.GetEnv'URL'))
      200≠r.HttpStatus:r.HttpMessage ⎕SIGNAL r.HttpStatus
     ⍝ ←: id  youtube_id  title  description  channel_id  channel  category_id  published_at  created_at  updated_at  updated_by
-     id←r.Data.items.id
-     _←{0=≢⍵:'' ⋄ ('YouTube videos ID= (',⍵,') not available.')⎕SIGNAL 404}1↓∊','∘,¨⍵/⍨~⍵∊id
+     0=⎕NC'r.Data.items':⎕SIGNAL⊂('EN'404)('Message' 'No items received from YouTube API')('Vendor' (#.DCMS.GetEnv'URL'))
+     id←{0∊⍴⍵:⍬ ⋄ ⍵.id}r.Data.items
+     _←{0=≢⍵:'' ⋄ ⎕SIGNAL⊂('EN'404)('Message' ('YouTube videos ID= (',⍵,') not available.'))('Vendor' (#.DCMS.GetEnv'URL'))}1↓∊','∘,¨⍵/⍨~⍵∊id
      res←'maxres' 'medium' 'default' 'high' 'standard'
      r.Data.items.thumbnail←res ChooseBestThumbnail r.Data.items
      ↑r.Data.items.((⊂id),snippet.(title description channelId channelTitle ⍬ publishedAt),⊂thumbnail)   ⍝ ⍬ for category_id

diff --git a/APLSource/read/videos/Query.aplf b/APLSource/read/videos/Query.aplf
@@ -1,4 +1,4 @@
- res←Query req;p;v;Q;CACHE;C;fm;to;ddn;in_range;i;data;pg;ppg;presenter;filter;event;Norm
+ res←Query req;C;CACHE;Norm;Q;Stem;data;ddn;event;filter;fm;i;in_range;p;pg;ppg;presenter;stemmer;terms;to;v
  (p v)←↓⍉req.QueryParams ⋄ v←RLTB¨v,⊂'' ⋄ v[⍸'null'∘≡¨v]←⊂'' ⋄ Q←{v⊃⍨p⍳⊆⍵}
  CACHE←##.##.CACHE
  C←CACHE.videos.(index_cols⊃⍨fields⍳⊂)
@@ -22,8 +22,14 @@
  filter←in_range∧presenter∧event
 
  Norm←##.##.Unidecode.NormaliseText
+ stemmer←⎕NEW #.DCMS.Porter2Stemmer.EnglishPorter2Stemmer
+ Stem←{
+     stemmed←stemmer.Stem⊆⍵
+     stemmed.Value
+ }
 ⍝ Rank by search terms
- i←(Norm Q'search')Rank filter⌿CACHE.videos.index_all
+ terms←Stem¨', '(~⍤∊⍨⊆⊢)Norm Q'search'
+ i←terms Rank filter⌿CACHE.videos.index_all
 
 ⍝ Sort
  data←(filter⌿CACHE.videos.values)[i;]

diff --git a/APLSource/read/videos/Rank.aplf b/APLSource/read/videos/Rank.aplf
@@ -1,11 +1,11 @@
  Rank←{
-⍝ ⍺: Simple char vec space- or comma-separated search terms
+⍝ ⍺: Nested list of search terms
 ⍝ ⍵: Text search index array
 ⍝ ←: Indices of rows in results
 ⍝    Only results which contain at least one instance of one of the search terms
 ⍝    Ranked descending term-frequency inverse-document-frequency
      0∊⍴⍺:⍳≢⍵   ⍝ No search terms provided, return all results
-     trm←', '(~⍤∊⍨⊆⊢)⍺
+     trm←⍺
      loc←trm⍷¨⊂⍵
      idx←(any←⊃∨/∨/¨loc)⌿⍵
      loc←any∘⌿¨loc

diff --git a/Admin/RunTests.aplf b/Admin/RunTests.aplf
@@ -0,0 +1,11 @@
+ r←RunTests keepalive;result
+ r←Tests.Run GetEnv'URL'
+ :If keepalive
+     :Return
+ :EndIf
+ :If 1≡r
+     ⎕←'All tests passed.',⎕UCS 10
+     ⎕OFF 0
+ :Else
+     ⎕OFF 1
+ :EndIf
diff --git a/Admin/Tests/DummyData/Test_GetStemmedWord.aplf b/Admin/Tests/DummyData/Test_GetStemmedWord.aplf
@@ -0,0 +1,9 @@
+ Test_GetStemmedWord←{
+⍝ Test free text search results include result from stemmed word
+     H←##.##.##.HttpCommand
+     url←⍵,'/videos?search=fugiativity'
+     res←H.GetJSON'GET'url
+     200≠res.HttpStatus:0
+     0∊⍴res.Data:0
+     'fugiat'(∨/⍷)1 ⎕JSON res.Data
+ }
diff --git a/Admin/Tests/DummyData/Test_GetStemmedWords.aplf b/Admin/Tests/DummyData/Test_GetStemmedWords.aplf
@@ -0,0 +1,10 @@
+ Test_GetStemmedWords←{
+⍝ Test free text search results include result from multiple stemmed words
+     H←##.##.##.HttpCommand
+     (stemmable stems)←↓⍉##.generator.stemmable
+     url←⍵,'/videos?search=',⊃(⊣,'%20',⊢)/stemmable
+     res←H.GetJSON'GET'url
+     200≠res.HttpStatus:0
+     0∊⍴res.Data:0
+     ∧/stems∘.(∨/⍷)⊂1 ⎕JSON res.Data
+ }
diff --git a/Admin/Tests/General/Test_Stemming.aplf b/Admin/Tests/General/Test_Stemming.aplf
@@ -0,0 +1,5 @@
+ Test_Stemming←{
+     stemmer←⎕NEW #.DCMS.Porter2Stemmer.EnglishPorter2Stemmer
+     word←stemmer.Stem⊂'words'
+     'word'≡word.Value
+ }
diff --git a/Admin/Tests/InsertDummyData.aplf b/Admin/Tests/InsertDummyData.aplf
@@ -1,4 +1,4 @@
- InsertDummyData;D;GEN;WrapBackticks;WrapColons;cid;col_spec;column;data;fn_cols;gen;idx;keywords;length;lipsum;n;name;opt;opt_cols;person_names;remove_columns;spec_fns;spec_opts;sql;table;thing_names;tid;type;type_spec;type_sql;types
+ InsertDummyData;D;NotFixedKeys;WrapBackticks;WrapColons;category;cid;col_spec;column;data;event;event_type;fixed_keys;keywords;lipsum;n;name;nvids;organisation;person;person_names;presentation;presentation_media;presentation_type;presenter;sql;stems;table;thing_names;tid;type;type_spec;videos;youtube_video
 ⍝ Insert dummy data into the database for testing
 ⍝ First tests considered:
 ⍝ - /videos?search= should return all videos with matching terms
@@ -8,9 +8,9 @@
 
 ⍝ Start by generating joined data and deconstructing it into the normalised database, then loop over each table and insert.
 
-
 ⍝ CONFIG
  keywords←generator.keywords
+ stems←generator.stemmable[;2]
  person_names←'Geoff' 'Streeter' 'John' 'Scholes' 'Iverson' 'Dolor' 'Fugiat' 'Orange' 'Rich' 'Park' 'Jada' 'Andrade' 'Andy' 'Karen' 'Fiona' 'Jason' 'Peter' 'Silas'
  thing_names←'APL' 'Meeting' 'Seeds' 'Conference' 'Dyalog' 'Alpha' 'Omega'
  lipsum←generator.lipsum
@@ -54,7 +54,8 @@
  presentation←()
  presentation.event_id←?n⍴n
  presentation.code←(vocab:⎕A,⎕D ⋄ max:2 ⋄ unique:1)generator.Name n
- presentation.title←(vocab:lipsum ⋄ max:30)generator.Text n
+ presentation.title←stems
+ presentation.title,←(vocab:lipsum ⋄ max:30)generator.Text n-≢stems
  presentation.title[1],←⊂' dolor'   ⍝ Ensure a keyword is present in search results
  presentation.type_id←?n⍴n
  presentation.description←(vocab:lipsum ⋄ max:65535)generator.Text n
@@ -63,16 +64,21 @@
  presentation.⎕DF'presentation'
 
  presenter←()
- presenter.(presentation_id person_id)←↓⍉{?⍵⍴⍛⍴n}⍣{∧/≠⍵}n 2⍴0
+ ⍝ Using fixed keys in the presenter and presentation_media tables ensures that stems are found in video results for testing
+ presenter.(presentation_id person_id)←fixed_keys←(1 2)(1 2)
+ NotFixedKeys←{~∨/,(↑fixed_keys)⍷⍤1 2⊢⍵}
+ presenter.(presentation_id person_id),←↓⍉{?⍵⍴⍛⍴n}⍣{(NotFixedKeys ⍺)∧∧/≠⍺}(n-≢fixed_keys)2⍴0
  presenter.⎕DF'presenter'
 
  ⍝ Videos is not a table to insert, but its data is used in other tables and must be consistent, so we create it here and re-use it below
  videos←()
  videos.presentation_id←?n⍴n
- videos.youtube_id←↓{(⎕A,⎕C ⎕A)[?⍵⍴⍛⍴2×26]}⍣{∧/≠⍵}n 11⍴0
+ videos.youtube_id←↓{(⎕A,⎕C ⎕A)[?⍵⍴⍛⍴2×26]}⍣{∧/≠⍺}n 11⍴0
+ nvids←≢∪videos.youtube_id
 
  presentation_media←(type:n⍴⊂'youtube_video')
- presentation_media.(presentation_id media_id)←↓⍉{?⍵⍴⍛⍴n}⍣{∧/≠⍵}n 2⍴0   ⍝ Random unique pairs of numbers
+ presentation_media.(presentation_id media_id)←fixed_keys←(1 2)(1 2)
+ presentation_media.(presentation_id media_id),←↓⍉{?⍵⍴⍛⍴n nvids}⍣{(NotFixedKeys ⍺)∧∧/≠⍺}(n-≢fixed_keys)2⍴0   ⍝ Random unique pairs of numbers
  presentation_media.⎕DF'presentation_media'
 
  youtube_video←(

diff --git a/Admin/Tests/generator/stemmable.apla b/Admin/Tests/generator/stemmable.apla
@@ -0,0 +1,4 @@
+[
+ 'ingenuity' 'ingenu'
+ 'amicable'  'amic'
+]
diff --git a/CI/activate.apls b/CI/activate.apls
@@ -0,0 +1,2 @@
+#!/usr/bin/dyalogscript DYALOG_INITSESSION=1
+⎕←⎕SE.UCMD'activate all -reset'
diff --git a/CI/install.apls b/CI/install.apls
@@ -0,0 +1,18 @@
+#!/usr/bin/dyalogscript DYALOG_INITSESSION=1
+⎕←'Installing application dependencies...'
+
+root←2 ⎕NQ # 'GetEnvironment' 'APP_DIR'
+pkg←root,'/packages'
+nuget←root,'/nuget-packages'
+
+:Trap 0
+    ⎕SE.Tatin.ReInstallDependencies pkg
+    ⎕SE.Tatin.LoadPackages 'NuGet' ⎕THIS
+    NuGet.Publish nuget
+    ⎕←'Application dependencies installed in ',pkg,' and ',nuget
+    ⎕OFF 0
+:Else
+    ⎕←'Could not install dependencies.'
+    ⎕←1⎕JSON⎕OPT'Compact'0⊢⎕DMX
+    ⎕OFF 1
+:EndTrap
diff --git a/CI/run-tests-in-docker.sh b/CI/run-tests-in-docker.sh
@@ -7,8 +7,11 @@ else
 fi
 
 ## Populate env file
-echo CONFIGFILE=/app/CI/testing.dcfg >> ${PWD}/env
-echo RIDE_INIT=HTTP:*:4502 >> ${PWD}/env
+rm ${PWD}/env
+echo YOUTUBE=http://localhost:8088/ >> ${PWD}/env
+echo APP_DIR=/app >> ${PWD}/env
+echo CONFIGFILE=/app/CI/test.dcfg >> ${PWD}/env
+echo RIDE_INIT=SERVE:*:4502 >> ${PWD}/env
 echo SQL_SERVER=db >> ${PWD}/env
 echo SQL_DATABASE=dyalog_cms >> ${PWD}/env
 echo SQL_USER=dcms >> ${PWD}/env
@@ -26,4 +29,4 @@ echo COMPOSE IS: $COMPOSE
 echo "Use docker inspect to get the IP of the running container"
 
 $COMPOSE pull
-$COMPOSE -f docker-compose.yml up
+$COMPOSE -f docker-compose.yml up db web --force-recreate --abort-on-container-exit
diff --git a/CI/test.dcfg b/CI/test.dcfg
@@ -0,0 +1,10 @@
+{
+    Extend: "../dcms.dcfg",
+    Settings: {
+        LX: "⎕TRAP ← 0 'C' '⎕←''Error during setup for test run'' ⋄ ⎕←1⎕JSON⎕OPT''Compact''0⊢⎕DMX ⋄ ⎕OFF 1' ⋄\
+             ⎕SE.Link.Import 'DCMS' '[APP_DIR]/APLSource' ⋄\
+             ⎕SE.Link.Import 'Admin' '[APP_DIR]/Admin' ⋄\
+             DCMS.Setup ⋄ DCMS.Run [DEBUG] ⋄\
+             Admin.RunTests 0"
+    }
+}
diff --git a/CI/testing.dcfg b/CI/testing.dcfg
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,7 @@
+FROM dyalog/techpreview:latest
+
+USER root
+RUN /tmp/dotnet-install.sh -c 8.0 -i /opt/dotnet
+RUN apt-get update && apt-get install -y zip && apt-get clean && rm -Rf /var/lib/apt/lists/*
+
+USER dyalog
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -6,23 +6,36 @@ def DockerDB
 def DockerDyalog
 def Testfile = "/tmp/dcms-CI.log"
 def Branch = env.BRANCH_NAME.toLowerCase()
+def Home = "/app/home"
 
 node ('Docker') {
 	stage ('Checkout') {
 		checkout scm
 	}
-	stage ('Update Dyalog') {
+	stage ('Build container') {
 		withDockerRegistry(credentialsId: '0435817a-5f0f-47e1-9dcc-800d85e5c335') {
-			DockerDyalog=docker.image('dyalog/techpreview:latest')
-			DockerDyalog.pull()
+			DockerDyalog = docker.build("dcms-build", "-f $WORKSPACE/Dockerfile .")
 		}
 	}
 	stage ('Update MariaDB') {
-			withDockerRegistry(credentialsId: '0435817a-5f0f-47e1-9dcc-800d85e5c335') {
+		withDockerRegistry(credentialsId: '0435817a-5f0f-47e1-9dcc-800d85e5c335') {
 			DockerDB=docker.image('mariadb:10.8.2') // Until build machine is updated
 			DockerDB.pull()
 		}
 	}
+	stage ('Install dependencies') {
+		sh "mkdir -p $WORKSPACE/home"
+		try {
+			DockerDyalog.inside("-t -u 6203 -v $WORKSPACE:/app -e HOME=${Home} -e APP_DIR=/app -e DOTNET_ROOT=/opt/dotnet --entrypoint=''"){
+				sh "/app/CI/activate.apls"
+				sh "/app/CI/install.apls"
+			}
+		} catch(e) {
+			println 'Could not install Tatin or NuGet dependencies.'
+			sh "docker rmi dcms-build"
+			throw new Exception("${e}")
+		}
+	}
 	stage ('Test service') {
 		DockerAppDB = DockerDB.run ("-e MYSQL_RANDOM_ROOT_PASSWORD=true -e MYSQL_DATABASE=dyalog_cms -e MYSQL_USER=dcms -e MYSQL_PASSWORD=apl")
 
@@ -37,7 +50,7 @@ node ('Docker') {
 		withCredentials([file(credentialsId: '205bc57d-1fae-4c67-9aeb-44c1144f071c', variable: 'DCMS_SECRETS')]) {
 
 			try {
-				DockerApp = DockerDyalog.run ("-t -u 6203 -v $DCMS_SECRETS:$DCMS_SECRETS -e HOME=/tmp -e CONFIGFILE=/app/CI/testing.dcfg -e SECRETS=$DCMS_SECRETS -e SQL_SERVER=${DBIP} -e SQL_DATABASE=dyalog_cms -e SQL_USER=dcms -e SQL_PASSWORD=apl -e SQL_PORT=3306 -v $WORKSPACE:/app")
+				DockerApp = DockerDyalog.run ("-t -u 6203 -v $DCMS_SECRETS:$DCMS_SECRETS -e HOME=${Home} -e CONFIGFILE=/app/CI/test.dcfg -e APP_DIR=/app -e YOUTUBE=http://localhost:8088/ -e SECRETS=$DCMS_SECRETS -e SQL_SERVER=${DBIP} -e SQL_DATABASE=dyalog_cms -e SQL_USER=dcms -e SQL_PASSWORD=apl -e SQL_PORT=3306 -v $WORKSPACE:/app")
 				println(DockerApp.id)
 				sh "docker logs -f ${DockerApp.id}"
 				def out = sh script: "docker inspect ${DockerApp.id} --format='{{.State.ExitCode}}'", returnStdout: true
@@ -53,6 +66,7 @@ node ('Docker') {
 					sh "${WORKSPACE}/CI/githubComment.sh ${DockerApp.id} ${commit_id}"
 				}
 				DockerApp.stop()
+				sh "docker rmi dcms-build"
 				echo "Throwing Exception..."
 				echo "Exception is: ${e}"
 				throw new Exception("${e}");
@@ -101,12 +115,12 @@ node ('Docker') {
 			echo SQL_PASSWORD=apl >> ${WORKSPACE}/env
 			echo SQL_PORT=3306 >> ${WORKSPACE}/env
 			echo SECRETS=/app/secrets/secrets.json5 >> ${WORKSPACE}/env
-			echo RIDE_INIT=http:*:4502 >> ${WORKSPACE}/env
 			echo MYSQL_DATABASE=dyalog_cms >> ${WORKSPACE}/env
 			echo MYSQL_USER=dcms >> ${WORKSPACE}/env
 			echo MYSQL_PASSWORD=apl >> ${WORKSPACE}/env
 			echo MYSQL_PORT=3306 >> ${WORKSPACE}/env
-			echo CONFIGFILE=/app/run.dcfg >> ${WORKSPACE}/env
+			echo YOUTUBE=https://www.googleapis.com/youtube/v3/ >> ${WORKSPACE}/env
+			echo APP_DIR=/app
 
 			echo MYSQL_RANDOM_ROOT_PASSWORD=1 >> ${WORKSPACE}/env
 		'''

diff --git a/dcms.dcfg b/dcms.dcfg
@@ -0,0 +1,18 @@
+{
+	Settings: {
+		MAXWS:        "2G",
+		PW:            300,
+		SERVICE_URL:  "http://localhost",
+		SERVICE_PORT:  8080,
+		URL:          "[SERVICE_URL]:[SERVICE_PORT]",
+		DEBUG:         0,
+		log_file:     "[APP_DIR]/dyalog_log_file.dlf",
+		SCHEMA_DEFS:  "[APP_DIR]/sql/*.sql",
+
+		PKG:          "[APP_DIR]/packages",
+		PKG_NUGET:    "[APP_DIR]/nuget-packages",
+
+		LX:           "⎕SE.Link.Import 'DCMS' '[APP_DIR]/APLSource' ⋄\
+					   DCMS.Setup ⋄ DCMS.Run [DEBUG]"
+	}
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		#!/usr/bin/dyalogscript DYALOG_INITSESSION=1
		⎕←⎕SE.UCMD'activate all -reset'