feat: implement the logic for running python stored programs

README.md

@@ -64,12 +64,27 @@ Example contents of a `.env` file:
 # Server mode is multi-user and suitable for intranet / internet use
 MODE=
 
+# A comma separated string that defines the available runTimes.
+# Priority is given to the runtime that comes first in the string.
+# Possible options at the moment are sas and js
+
+# This string sets the priority of the available analytic runtimes
+# Valid runtimes are SAS (sas), JavaScript (js) and Python (py)
+# For each option provided, there should be a corresponding path,
+# eg SAS_PATH, NODE_PATH or PYTHON_PATH
+# Priority is given to runtimes earlier in the string
+# Example options:  [sas,js,py | js,py | sas | sas,js]
+RUN_TIMES=
+
 # Path to SAS executable (sas.exe / sas.sh)
 SAS_PATH=/path/to/sas/executable.exe
 
 # Path to Node.js executable
 NODE_PATH=~/.nvm/versions/node/v16.14.0/bin/node
 
+# Path to Python executable
+PYTHON_PATH=/usr/bin/python
+
 # Path to working directory
 # This location is for SAS WORK, staged files, DRIVE, configuration etc
 SASJS_ROOT=./sasjs_root
@@ -139,13 +154,6 @@ LOG_FORMAT_MORGAN=
 # This location is for server logs with classical UNIX logrotate behavior
 LOG_LOCATION=./sasjs_root/logs
 
-# A comma separated string that defines the available runTimes.
-# Priority is given to the runtime that comes first in the string.
-# Possible options at the moment are sas and js
-
-# options: [sas,js|js,sas|sas|js] default:sas
-RUN_TIMES=
-
 ```
 
 ## Persisting the Session

api/.env.example

@@ -14,9 +14,10 @@ HELMET_COEP=[true|false] if omitted HELMET default will be used
 
 DB_CONNECT=mongodb+srv://<DB_USERNAME>:<DB_PASSWORD>@<CLUSTER>/<DB_NAME>?retryWrites=true&w=majority
 
-RUN_TIMES=[sas|js|sas,js|js,sas] default considered as sas
+RUN_TIMES=[sas,js,py | js,py | sas | sas,js] default considered as sas
 SAS_PATH=/opt/sas/sas9/SASHome/SASFoundation/9.4/sas
 NODE_PATH=~/.nvm/versions/node/v16.14.0/bin/node
+PYTHON_PATH=/usr/bin/python
 
 SASJS_ROOT=./sasjs_root
 

api/src/controllers/internal/Execution.ts

@@ -89,13 +89,16 @@ export class ExecutionController {
       tokenFile,
       preProgramVariables?.httpHeaders.join('\n') ?? ''
     )
+    if (returnJson)
+      await createFile(headersPath, 'Content-type: application/json')
 
     await processProgram(
       program,
       preProgramVariables,
       vars,
       session,
       weboutPath,
+      headersPath,
       tokenFile,
       runTime,
       logPath,

api/src/controllers/internal/Session.ts

@@ -190,7 +190,39 @@ export class JSSessionController extends SessionController {
     }
 
     const headersPath = path.join(session.path, 'stpsrv_header.txt')
-    await createFile(headersPath, 'Content-type: application/json')
+    await createFile(headersPath, 'Content-type: text/plain')
+
+    this.sessions.push(session)
+    return session
+  }
+}
+
+export class PythonSessionController extends SessionController {
+  protected async createSession(): Promise<Session> {
+    const sessionId = generateUniqueFileName(generateTimestamp())
+    const sessionFolder = path.join(getSessionsFolder(), sessionId)
+
+    const creationTimeStamp = sessionId.split('-').pop() as string
+    // death time of session is 15 mins from creation
+    const deathTimeStamp = (
+      parseInt(creationTimeStamp) +
+      15 * 60 * 1000 -
+      1000
+    ).toString()
+
+    const session: Session = {
+      id: sessionId,
+      ready: true,
+      inUse: true,
+      consumed: false,
+      completed: false,
+      creationTimeStamp,
+      deathTimeStamp,
+      path: sessionFolder
+    }
+
+    const headersPath = path.join(session.path, 'stpsrv_header.txt')
+    await createFile(headersPath, 'Content-type: text/plain')
 
     this.sessions.push(session)
     return session
@@ -199,7 +231,7 @@ export class JSSessionController extends SessionController {
 
 export const getSessionController = (
   runTime: RunTimeType
-): SASSessionController | JSSessionController => {
+): SASSessionController | JSSessionController | PythonSessionController => {
   if (runTime === RunTimeType.SAS) {
     return getSASSessionController()
   }
@@ -208,6 +240,10 @@ export const getSessionController = (
     return getJSSessionController()
   }
 
+  if (runTime === RunTimeType.PY) {
+    return getPythonSessionController()
+  }
+
   throw new Error('No Runtime is configured')
 }
 
@@ -227,6 +263,14 @@ const getJSSessionController = (): JSSessionController => {
   return process.jsSessionController
 }
 
+const getPythonSessionController = (): PythonSessionController => {
+  if (process.pythonSessionController) return process.pythonSessionController
+
+  process.pythonSessionController = new PythonSessionController()
+
+  return process.pythonSessionController
+}
+
 const autoExecContent = `
 data _null_;
   /* remove the dummy SYSIN */

api/src/controllers/internal/createJSProgram.ts

@@ -9,6 +9,7 @@ export const createJSProgram = async (
   vars: ExecutionVars,
   session: Session,
   weboutPath: string,
+  headersPath: string,
   tokenFile: string,
   otherArgs?: any
 ) => {
@@ -23,15 +24,16 @@ let _webout = '';
 const weboutPath = '${
     isWindows() ? weboutPath.replace(/\\/g, '\\\\') : weboutPath
   }'; 
-const _sasjs_tokenfile = '${
+const _SASJS_TOKENFILE = '${
     isWindows() ? tokenFile.replace(/\\/g, '\\\\') : tokenFile
   }';
-const _sasjs_username = '${preProgramVariables?.username}';
-const _sasjs_userid = '${preProgramVariables?.userId}';
-const _sasjs_displayname = '${preProgramVariables?.displayName}';
-const _metaperson = _sasjs_displayname;
-const _metauser = _sasjs_username;
-const sasjsprocessmode = 'Stored Program';
+const _SASJS_WEBOUT_HEADERS = '${headersPath}';
+const _SASJS_USERNAME = '${preProgramVariables?.username}';
+const _SASJS_USERID = '${preProgramVariables?.userId}';
+const _SASJS_DISPLAYNAME = '${preProgramVariables?.displayName}';
+const _METAPERSON = _SASJS_DISPLAYNAME;
+const _METAUSER = _SASJS_USERNAME;
+const SASJSPROCESSMODE = 'Stored Program';
 `
 
   const requiredModules = `const fs = require('fs')`
@@ -55,14 +57,15 @@ if (_webout) {
 `
   // if no files are uploaded filesNamesMap will be undefined
   if (otherArgs?.filesNamesMap) {
-    const uploadJSCode = await generateFileUploadJSCode(
+    const uploadJsCode = await generateFileUploadJSCode(
       otherArgs.filesNamesMap,
       session.path
     )
 
-    //If js code for the file is generated it will be appended to the top of jsCode
-    if (uploadJSCode.length > 0) {
-      program = `${uploadJSCode}\n` + program
+    // If any files are uploaded, the program needs to be updated with some
+    // dynamically generated variables (pointers) for ease of ingestion
+    if (uploadJsCode.length > 0) {
+      program = `${uploadJsCode}\n` + program
     }
   }
   return requiredModules + program

api/src/controllers/internal/createPythonProgram.ts

@@ -0,0 +1,68 @@
+import { isWindows } from '@sasjs/utils'
+import { PreProgramVars, Session } from '../../types'
+import { generateFileUploadPythonCode } from '../../utils'
+import { ExecutionVars } from './'
+
+export const createPythonProgram = async (
+  program: string,
+  preProgramVariables: PreProgramVars,
+  vars: ExecutionVars,
+  session: Session,
+  weboutPath: string,
+  headersPath: string,
+  tokenFile: string,
+  otherArgs?: any
+) => {
+  const varStatments = Object.keys(vars).reduce(
+    (computed: string, key: string) => `${computed}${key} = '${vars[key]}';\n`,
+    ''
+  )
+
+  const preProgramVarStatments = `
+_SASJS_SESSION_PATH = '${
+    isWindows() ? session.path.replace(/\\/g, '\\\\') : session.path
+  }';
+_WEBOUT = '${isWindows() ? weboutPath.replace(/\\/g, '\\\\') : weboutPath}'; 
+_SASJS_WEBOUT_HEADERS = '${headersPath}';
+_SASJS_TOKENFILE = '${
+    isWindows() ? tokenFile.replace(/\\/g, '\\\\') : tokenFile
+  }';
+_SASJS_USERNAME = '${preProgramVariables?.username}';
+_SASJS_USERID = '${preProgramVariables?.userId}';
+_SASJS_DISPLAYNAME = '${preProgramVariables?.displayName}';
+_METAPERSON = _SASJS_DISPLAYNAME;
+_METAUSER = _SASJS_USERNAME;
+SASJSPROCESSMODE = 'Stored Program';
+`
+
+  const requiredModules = `import os`
+
+  program = `
+# runtime vars
+${varStatments}
+
+# dynamic user-provided vars
+${preProgramVarStatments}
+
+# change working directory to  session folder
+os.chdir(_SASJS_SESSION_PATH)
+
+# actual job code
+${program}
+
+`
+  // if no files are uploaded filesNamesMap will be undefined
+  if (otherArgs?.filesNamesMap) {
+    const uploadPythonCode = await generateFileUploadPythonCode(
+      otherArgs.filesNamesMap,
+      session.path
+    )
+
+    // If any files are uploaded, the program needs to be updated with some
+    // dynamically generated variables (pointers) for ease of ingestion
+    if (uploadPythonCode.length > 0) {
+      program = `${uploadPythonCode}\n` + program
+    }
+  }
+  return requiredModules + program
+}

api/src/controllers/internal/createSASProgram.ts

@@ -23,10 +23,14 @@ export const createSASProgram = async (
 %let _sasjs_displayname=${preProgramVariables?.displayName};
 %let _sasjs_apiserverurl=${preProgramVariables?.serverUrl};
 %let _sasjs_apipath=/SASjsApi/stp/execute;
+%let _sasjs_webout_headers=%sysfunc(pathname(work))/../stpsrv_header.txt;
 %let _metaperson=&_sasjs_displayname;
 %let _metauser=&_sasjs_username;
+
+/* the below is here for compatibility and will be removed in a future release */
+%let sasjs_stpsrv_header_loc=&_sasjs_webout_headers;
+
 %let sasjsprocessmode=Stored Program;
-%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/../stpsrv_header.txt;
 
 %global SYSPROCESSMODE SYSTCPIPHOSTNAME SYSHOSTINFOLONG;
 %macro _sasjs_server_init();
@@ -63,7 +67,8 @@ ${program}`
       session.path
     )
 
-    //If sas code for the file is generated it will be appended to the top of sasCode
+    // If any files are uploaded, the program needs to be updated with some
+    // dynamically generated variables (pointers) for ease of ingestion
     if (uploadSasCode.length > 0) {
       program = `${uploadSasCode}` + program
     }

api/src/controllers/internal/index.ts

@@ -4,4 +4,5 @@ export * from './Execution'
 export * from './FileUploadController'
 export * from './createSASProgram'
 export * from './createJSProgram'
+export * from './createPythonProgram'
 export * from './processProgram'

api/src/controllers/internal/processProgram.ts

@@ -5,14 +5,20 @@ import { once } from 'stream'
 import { createFile, moveFile } from '@sasjs/utils'
 import { PreProgramVars, Session } from '../../types'
 import { RunTimeType } from '../../utils'
-import { ExecutionVars, createSASProgram, createJSProgram } from './'
+import {
+  ExecutionVars,
+  createSASProgram,
+  createJSProgram,
+  createPythonProgram
+} from './'
 
 export const processProgram = async (
   program: string,
   preProgramVariables: PreProgramVars,
   vars: ExecutionVars,
   session: Session,
   weboutPath: string,
+  headersPath: string,
   tokenFile: string,
   runTime: RunTimeType,
   logPath: string,
@@ -25,6 +31,7 @@ export const processProgram = async (
       vars,
       session,
       weboutPath,
+      headersPath,
       tokenFile,
       otherArgs
     )
@@ -47,6 +54,43 @@ export const processProgram = async (
       // copy the code.js program to log and end write stream
       writeStream.end(program)
 
+      session.completed = true
+      console.log('session completed', session)
+    } catch (err: any) {
+      session.completed = true
+      session.crashed = err.toString()
+      console.log('session crashed', session.id, session.crashed)
+    }
+  } else if (runTime === RunTimeType.PY) {
+    program = await createPythonProgram(
+      program,
+      preProgramVariables,
+      vars,
+      session,
+      weboutPath,
+      headersPath,
+      tokenFile,
+      otherArgs
+    )
+
+    const codePath = path.join(session.path, 'code.py')
+
+    try {
+      await createFile(codePath, program)
+
+      // create a stream that will write to console outputs to log file
+      const writeStream = fs.createWriteStream(logPath)
+
+      // waiting for the open event so that we can have underlying file descriptor
+      await once(writeStream, 'open')
+
+      execFileSync(process.pythonLoc!, [codePath], {
+        stdio: ['ignore', writeStream, writeStream]
+      })
+
+      // copy the code.py program to log and end write stream
+      writeStream.end(program)
+
       session.completed = true
       console.log('session completed', session)
     } catch (err: any) {