upload via post

package.json

@@ -64,6 +64,7 @@
     "cidr-tools": "^4.3.0",
     "fast-glob": "^3.2.7",
     "find-process": "^1.4.7",
+    "formidable": "^2.1.1",
     "koa": "^2.13.4",
     "koa-compress": "^5.1.0",
     "koa-mount": "^4.0.0",
@@ -79,6 +80,7 @@
   "devDependencies": {
     "@types/archiver": "^5.1.1",
     "@types/basic-auth": "^1.1.3",
+    "@types/formidable": "^2.0.5",
     "@types/koa": "^2.13.4",
     "@types/koa__router": "^8.0.11",
     "@types/koa-compress": "^4.0.3",

plugins/vhosting/plugin.js

@@ -24,7 +24,7 @@ exports.init = api => {
             let toModify = ctx
             if (ctx.path.startsWith(api.const.SPECIAL_URI)) { // special uris should be excluded...
                 toModify = ctx.params
-                if (toModify.path === undefined) // ...unless they carry a path in the query. In that case we'll work that.
+                if (toModify?.path === undefined) // ...unless they carry a path in the query. In that case we'll work that.
                     return
             }
             const hosts = api.getConfig('hosts')

src/apiMiddleware.ts

@@ -3,7 +3,7 @@
 import Koa from 'koa'
 import createSSE from './sse'
 import { Readable } from 'stream'
-import { asyncGeneratorToReadable, onOff } from './misc'
+import { asyncGeneratorToReadable, objSameKeys, onOff, stream2string, tryJson } from './misc'
 import events from './events'
 import { HTTP_BAD_REQUEST, HTTP_NOT_FOUND, HTTP_UNAUTHORIZED } from './const'
 import _, { DebouncedFunc } from 'lodash'
@@ -26,10 +26,13 @@ export function apiMiddleware(apis: ApiHandlers) : Koa.Middleware {
             ctx.body = 'invalid api'
             return ctx.status = HTTP_NOT_FOUND
         }
+        ctx.params = ctx.method === 'POST' ? tryJson(await stream2string(ctx.req))
+            : objSameKeys(ctx.query, x => Array.isArray(x) ? x : tryJson(x))
+        console.debug('API', ctx.method, ctx.path, { ...ctx.params })
         const csrf = ctx.cookies.get('csrf')
         // we don't rely on SameSite cookie option because it's https-only
-        let res = csrf && csrf !== params.csrf ? new ApiError(HTTP_UNAUTHORIZED, 'csrf')
-            : await apiFun(params || {}, ctx)
+        let res = csrf && csrf !== ctx.params.csrf ? new ApiError(HTTP_UNAUTHORIZED, 'csrf')
+            : await apiFun(ctx.params || {}, ctx)
         if (isAsyncGenerator(res))
             res = asyncGeneratorToReadable(res)
         if (res instanceof Readable) { // Readable, we'll go SSE-mode

src/index.ts

@@ -9,14 +9,15 @@ import { frontEndApis } from './frontEndApis'
 import { log } from './log'
 import { pluginsMiddleware } from './plugins'
 import { throttler } from './throttler'
-import { headRequests, gzipper, sessions, serveGuiAndSharedFiles, someSecurity, prepareState, paramsDecoder } from './middlewares'
+import { headRequests, gzipper, sessions, serveGuiAndSharedFiles, someSecurity, prepareState } from './middlewares'
 import './listen'
 import './commands'
 import { adminApis } from './adminApis'
 import { defineConfig } from './config'
 import { ok } from 'assert'
 import _ from 'lodash'
 import { randomId } from './misc'
+//import body from 'koa-better-body'
 
 ok(_.intersection(Object.keys(frontEndApis), Object.keys(adminApis)).length === 0) // they share same endpoints
 
@@ -29,9 +30,9 @@ app.use(someSecurity)
     .use(log())
     .use(throttler)
     .use(gzipper)
-    .use(paramsDecoder)
     .use(pluginsMiddleware())
     .use(mount(API_URI, apiMiddleware({ ...frontEndApis, ...adminApis })))
+    //.use(body({ multipart: false }))
     .use(serveGuiAndSharedFiles)
     .on('error', errorHandler)
 

src/middlewares.ts

@@ -4,7 +4,6 @@ import compress from 'koa-compress'
 import Koa from 'koa'
 import session from 'koa-session'
 import {
-    HTTP_FOOL,
     ADMIN_URI,
     BUILD_TIMESTAMP,
     DEV,
@@ -15,21 +14,22 @@ import {
 } from './const'
 import { FRONTEND_URI } from './const'
 import { cantReadStatusCode, hasPermission, nodeIsDirectory, urlToNode, vfs, VfsNode } from './vfs'
-import { dirTraversal, objSameKeys, prepareFolder, tryJson } from './misc'
+import { dirTraversal } from './misc'
 import { zipStreamFromFolder } from './zip'
 import { serveFileNode } from './serveFile'
 import { serveGuiFiles } from './serveGuiFiles'
 import mount from 'koa-mount'
-import { Readable } from 'stream'
+import { once, Readable } from 'stream'
 import { applyBlock } from './block'
 import { getAccount } from './perm'
 import { socket2connection, updateConnection, normalizeIp } from './connections'
 import basicAuth from 'basic-auth'
 import { SRPClientSession, SRPParameters, SRPRoutines } from 'tssrp6a'
 import { srpStep1 } from './api.auth'
 import { basename, dirname, join } from 'path'
-import { createWriteStream } from 'fs'
+import { createWriteStream, mkdirSync } from 'fs'
 import { pipeline } from 'stream/promises'
+import formidable from 'formidable'
 
 export const gzipper = compress({
     threshold: 2048,
@@ -82,11 +82,27 @@ export const serveGuiAndSharedFiles: Koa.Middleware = async (ctx, next) => {
         const folder = await urlToNode(dirname(decPath), ctx, vfs, v => rest = v+'/'+rest)
         if (!folder)
             return ctx.status = HTTP_NOT_FOUND
-        return await receiveUpload(folder, rest, ctx.req, ctx)
+        const dest = uploadWriter(folder, rest, ctx)
+        if (dest) {
+            await pipeline(ctx.req, dest)
+            ctx.body = {}
+        }
+        return
     }
     const node = await urlToNode(path, ctx)
     if (!node)
         return ctx.status = HTTP_NOT_FOUND
+    if (ctx.method === 'POST') { // curl -F upload=@file url/
+        ctx.body = {}
+        const form = formidable({
+            maxFileSize: Infinity,
+            //@ts-ignore wrong in the .d.ts file
+            fileWriteStreamHandler: f => uploadWriter(node, f.originalFilename, ctx)
+        })
+        form.parse(ctx.req)
+        await once(form, 'end').catch(()=> {})
+        return
+    }
     const canRead = hasPermission(node, 'can_read', ctx)
     const isFolder = await nodeIsDirectory(node)
     if (isFolder && !path.endsWith('/'))
@@ -115,16 +131,17 @@ export const serveGuiAndSharedFiles: Koa.Middleware = async (ctx, next) => {
             : ctx.status = cantReadStatusCode(def)
     }
     return serveFrontendFiles(ctx, next)
-}
 
-async function receiveUpload(base: VfsNode, path: string, stream: Readable, ctx: Koa.Context) {
-    if (!base.source || !hasPermission(base, 'can_upload', ctx))
-        return ctx.status = base.can_upload === false ? HTTP_FORBIDDEN : HTTP_UNAUTHORIZED
-    path = join(base.source, path)
-    await prepareFolder(path)
-    const dest = createWriteStream(path)
-    await pipeline(stream, dest)
-    ctx.body = '{}'
+    function uploadWriter(base: VfsNode, path: string, ctx: Koa.Context) {
+        if (!base.source || !hasPermission(base, 'can_upload', ctx)) {
+            ctx.status = base.can_upload === false ? HTTP_FORBIDDEN : HTTP_UNAUTHORIZED
+            return
+        }
+        path = join(base.source, path)
+        mkdirSync(dirname(path), { recursive: true })
+        return createWriteStream(path)
+    }
+
 }
 
 let proxyDetected = false
@@ -136,14 +153,14 @@ export const someSecurity: Koa.Middleware = async (ctx, next) => {
         if (DEV && proxy && [process.env.FRONTEND_PROXY, process.env.ADMIN_PROXY].includes(ctx.get('X-Forwarded-port')))
             proxy = ''
         if (dirTraversal(decodeURI(ctx.path)))
-            return ctx.status = HTTP_FOOL
+            return ctx.status = 418
         if (applyBlock(ctx.socket, ctx.ip))
             return
         proxyDetected ||= proxy > ''
         ctx.state.proxiedFor = proxy
     }
     catch {
-        return ctx.status = HTTP_FOOL
+        return ctx.status = 418
     }
     return next()
 }
@@ -177,27 +194,3 @@ async function srpCheck(username: string, password: string) {
     const clientRes2 = await clientRes1.step2(BigInt(salt), BigInt(pubKey))
     return await step1.step2(clientRes2.A, clientRes2.M1).then(() => true, () => false)
 }
-
-// unify get/post parameters, with JSON decoding to not be limited to strings
-export const paramsDecoder: Koa.Middleware = async (ctx, next) => {
-    ctx.params = ctx.method === 'POST' ? tryJson(await stream2string(ctx.req))
-        : objSameKeys(ctx.query, x => Array.isArray(x) ? x : tryJson(x))
-    await next()
-}
-
-async function stream2string(stream: Readable): Promise<string> {
-    return new Promise((resolve, reject) => {
-        let data = ''
-        stream.on('data', chunk =>
-            data += chunk)
-        stream.on('error', reject)
-        stream.on('end', () => {
-            try {
-                resolve(data)
-            }
-            catch(e) {
-                reject(e)
-            }
-        })
-    })
-}

src/misc.ts

@@ -10,6 +10,7 @@ export * from './util-http'
 export * from './util-generators'
 export * from './util-files'
 import debounceAsync from './debounceAsync'
+import { Readable } from 'stream'
 export { debounceAsync }
 
 export type Callback<IN=void, OUT=void> = (x:IN) => OUT
@@ -147,3 +148,20 @@ export function tryJson(s?: string) {
     try { return s && JSON.parse(s) }
     catch {}
 }
+
+export async function stream2string(stream: Readable): Promise<string> {
+    return new Promise((resolve, reject) => {
+        let data = ''
+        stream.on('data', chunk =>
+            data += chunk)
+        stream.on('error', reject)
+        stream.on('end', () => {
+            try {
+                resolve(data)
+            }
+            catch(e) {
+                reject(e)
+            }
+        })
+    })
+}

todo.md

@@ -1,4 +1,5 @@
 # To do
+- upload: minimum available disk
 - admin/fs: check if source exist when set
 - plugins: after installing, switch to installed (and perhaps highlight new one)
 - plugins' log, accessible in admin
@@ -36,7 +37,8 @@
 - remove seconds from time
 - upload
 - upload unzipping (while streaming?)
-- plugin to make letsencrypt easier
+- plugin: upload quota per-account (possibly inheriting), and a default
+- plugin: make letsencrypt easier
   - could be just automatic detection of files by certbot
   - letsencrypt supports plugins to automatically configure webservers
 - delete

tsconfig.json

@@ -25,7 +25,7 @@
     // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
 
     /* Modules */
-    "module": "commonjs",                                /* Specify what module code is generated. */
+    "module": "node16",                                /* Specify what module code is generated. */
     "rootDir": "src",                                  /* Specify the root folder within your source files. */
     // "moduleResolution": "node",                       /* Specify how TypeScript looks up a file from a given module specifier. */
     // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */