enso/tools/simple-library-server/main.js

#!/usr/bin/env node
const express = require('express')
const crypto = require('crypto')
const path = require('path')
const os = require('os')
const fs = require('fs')
const fsPromises = require('fs/promises')
const multer = require('multer')
const compression = require('compression')
const yargs = require('yargs')
const semverValid = require('semver/functions/valid')

const LOG_REQUESTS = false

const argv = yargs
    .usage(
        '$0',
        'Allows to host Enso libraries and editions from the local filesystem through HTTP.'
    )
    .option('port', {
        description: 'The port to listen on.',
        type: 'number',
        default: 8080,
    })
    .option('root', {
        description:
            'The root of the repository. It should contain a `libraries` or `editions` directory. See the documentation for more details.',
        type: 'string',
        default: '.',
    })
    .option('upload', {
        description:
            'Specifies whether to allow uploading libraries and which authentication model to choose.',
        choices: ['disabled', 'no-auth', 'constant-token'],
        default: 'disabled',
    })
    .help()
    .alias('help', 'h').argv

const libraryRoot = path.join(argv.root, 'libraries')

const app = express()
if (LOG_REQUESTS) {
    app.use((req, res, next) => {
        console.log(`Received [${req.method}] ${req.originalUrl}`)
        console.log(`  Headers: ${JSON.stringify(req.headers)}`)
        console.log(`  Query: ${JSON.stringify(req.query)}`)
        console.log(`  Body: ${JSON.stringify(req.body)}`)
        next()
    })
}

const tmpDir = path.join(os.tmpdir(), 'enso-library-repo-uploads')
const upload = multer({ dest: tmpDir })
app.use(compression({ filter: shouldCompress }))

/** The token to compare against for simple authentication.
 *
 * If it is not set, no authentication checks are made.
 */
let token = null
if (argv.upload == 'disabled') {
    console.log('Uploads are disabled.')
} else {
    app.post('/upload', upload.any(), handleUpload)

    if (argv.upload == 'constant-token') {
        const envVar = 'ENSO_AUTH_TOKEN'
        token = process.env[envVar]
        if (!token) {
            throw `${envVar} is not defined.`
        } else {
            console.log(`Checking the ${envVar} to authorize requests.`)
        }
    } else {
        console.log('WARNING: Uploads are enabled without any authentication.')
    }
}

app.get('/health', function (req, res) {
    res.status(200).send('OK')
})

app.use(express.static(argv.root))

let port = argv.port
if (process.env.PORT) {
    port = process.env.PORT
    console.log(`Overriding the port to ${port} set by the PORT environment variable.`)
}
console.log(`Serving the repository located under ${argv.root} on port ${port}.`)

const server = app.listen(port)

function handleShutdown() {
    console.log('Received a signal - shutting down.')
    server.close(() => {
        console.log('Server terminated.')
    })
}
process.on('SIGTERM', handleShutdown)
process.on('SIGINT', handleShutdown)

/// Specifies if a particular file can be compressed in transfer, if supported.
function shouldCompress(req, res) {
    if (req.path.endsWith('.yaml')) {
        return true
    }

    return compression.filter(req, res)
}

/** Handles upload of a library. */
async function handleUpload(req, res) {
    function fail(code, message) {
        res.status(code).json({ error: message })
        if (req.files !== undefined) {
            cleanFiles(req.files)
        }
    }

    if (token !== null) {
        const userToken = req.get('Auth-Token')
        if (userToken != token) {
            return fail(403, 'Authorization failed.')
        }
    }

    const version = req.query.version
    const namespace = req.query.namespace
    const name = req.query.name

    if (version === undefined || namespace == undefined || name === undefined) {
        return fail(400, 'One or more required fields were missing.')
    }

    if (!isVersionValid(version)) {
        return fail(400, `Invalid semver version string [${version}].`)
    }

    if (!isNamespaceValid(namespace)) {
        return fail(400, `Invalid username [${namespace}].`)
    }

    if (!isNameValid(name)) {
        return fail(400, `Invalid library name [${name}].`)
    }

    for (var i = 0; i < req.files.length; ++i) {
        const filename = req.files[i].originalname
        if (!isFilenameValid(filename)) {
            return fail(400, `Invalid filename: ${filename}.`)
        }
    }

    const libraryBasePath = path.join(libraryRoot, namespace, name)
    const libraryPath = path.join(libraryBasePath, version)

    /** Finds a name for a temporary directory to move the files to,
      so that the upload can then be committed atomically by renaming
      a single directory. */
    function findRandomTemporaryDirectory() {
        const randomName = crypto.randomBytes(32).toString('hex')
        const temporaryPath = path.join(libraryBasePath, randomName)
        if (fs.existsSync(temporaryPath)) {
            return findRandomTemporaryDirectory()
        }

        return temporaryPath
    }

    if (fs.existsSync(libraryPath)) {
        return fail(
            409,
            'A library with the given name and version ' +
                'combination already exists. Versions are immutable, so you must ' +
                'bump the library version when uploading a newer version.'
        )
    }

    const temporaryPath = findRandomTemporaryDirectory()
    await fsPromises.mkdir(libraryBasePath, { recursive: true })
    await fsPromises.mkdir(temporaryPath, { recursive: true })

    console.log(`Uploading library [${namespace}.${name}:${version}].`)
    try {
        await putFiles(temporaryPath, req.files)
        await fsPromises.rename(temporaryPath, libraryPath)
    } catch (error) {
        console.log(`Upload failed: [${error}].`)
        console.error(error.stack)
        return fail(500, 'Upload failed due to an internal error.')
    }

    console.log('Upload complete.')
    res.status(200).json({ message: 'Successfully uploaded the library.' })
}

/// Checks if a version complies with the semver specification.
function isVersionValid(version) {
    return semverValid(version) !== null
}

/// Checks if the namespace/username is valid.
function isNamespaceValid(namespace) {
    return /^[A-Za-z][a-z0-9]*$/.test(namespace) && namespace.length >= 3
}

/** Checks if the library name is valid.
 *
 * It may actually accept more identifiers as valid than Enso would, the actual
 * check should be done when creating the library. This is just a sanity check
 * for safety.
 */
function isNameValid(name) {
    return /^[A-Za-z0-9_]+$/.test(name)
}

// TODO [RW] for now slashes are not permitted to avoid attacks; later on at least the `meta` directory should be allowed, but not much besides that
/// Checks if the uploaded filename is valid.
function isFilenameValid(name) {
    return /^[A-Za-z0-9][A-Za-z0-9\._\-]*$/.test(name)
}

/// Schedules to remove the files, if they still exist.
function cleanFiles(files) {
    files.forEach(file => {
        if (fs.existsSync(file.path)) {
            fs.unlink(file.path, err => {
                if (err) {
                    console.error(
                        `Failed to remove ${file.path} ($file.originalname) from a failed upload: ${err}.`
                    )
                }
            })
        }
    })
}

/// Moves the files to the provided destination directory.
async function putFiles(directory, files) {
    for (var i = 0; i < files.length; ++i) {
        const file = files[i]
        const filename = file.originalname
        const destination = path.join(directory, filename)
        await fsPromises.copyFile(file.path, destination)
        await fsPromises.unlink(file.path)
    }
}