Enable upload of Linux packages to packagecloud from release builds

script/lib/upload-linux-packages.js

@@ -0,0 +1,112 @@
+const fs = require('fs')
+const path = require('path')
+const request = require('request-promise-native')
+
+module.exports = async function (packageRepoName, apiToken, version, artifacts) {
+  for (let artifact of artifacts) {
+    let fileExt = path.extname(artifact)
+    switch (fileExt) {
+      case '.deb':
+        await uploadDebPackage(version, artifact)
+        break
+      case '.rpm':
+        await uploadRpmPackage(version, artifact)
+        break
+      default:
+        continue
+    }
+  }
+
+  async function uploadDebPackage (version, filePath) {
+    // NOTE: Not sure if distro IDs update over time, might need
+    // to query the following endpoint dynamically to find the right IDs:
+    //
+    // https://{apiToken}:@packagecloud.io/api/v1/distributions.json
+    await uploadPackage({
+      version,
+      filePath,
+      type: 'deb',
+      arch: 'amd64',
+      fileName: 'atom-amd64.deb',
+      distroId: 35 /* Any .deb distribution */,
+      distroName: 'any',
+      distroVersion: 'any'
+    })
+  }
+
+  async function uploadRpmPackage (version, filePath) {
+    await uploadPackage({
+      version,
+      filePath,
+      type: 'rpm',
+      arch: 'x86_64',
+      fileName: 'atom.x86_64.rpm',
+      distroId: 140 /* Enterprise Linux 7 */,
+      distroName: 'el',
+      distroVersion: '7'
+    })
+  }
+
+  async function uploadPackage (packageDetails) {
+    // Infer the package suffix from the version
+    if (/-beta\d+/.test(packageDetails.version)) {
+      packageDetails.releaseSuffix = '-beta'
+    } else if (/-nightly\d+/.test(packageDetails.version)) {
+      packageDetails.releaseSuffix = '-nightly'
+    }
+
+    await removePackageIfExists(packageDetails)
+    await uploadToPackageCloud(packageDetails)
+  }
+
+  function uploadToPackageCloud (packageDetails) {
+    return new Promise(async (resolve, reject) => {
+      console.log(`Uploading ${packageDetails.fileName} to https://packagecloud.io/AtomEditor/${packageRepoName}`)
+      var uploadOptions = {
+        url: `https://${apiToken}:@packagecloud.io/api/v1/repos/AtomEditor/${packageRepoName}/packages.json`,
+        formData: {
+          'package[distro_version_id]': packageDetails.distroId,
+          'package[package_file]': fs.createReadStream(packageDetails.filePath)
+        }
+      }
+
+      request.post(uploadOptions, (error, uploadResponse, body) => {
+        if (error || uploadResponse.statusCode !== 201) {
+          console.log(`Error while uploading '${packageDetails.fileName}' v${packageDetails.version}: ${uploadResponse}`)
+          reject(uploadResponse)
+        } else {
+          console.log(`Successfully uploaded ${packageDetails.fileName}!`)
+          resolve(uploadResponse)
+        }
+      })
+    })
+  }
+
+  async function removePackageIfExists ({ version, type, arch, fileName, distroName, distroVersion, releaseSuffix }) {
+    // RPM URI paths have an extra '/0.1' thrown in
+    let versionJsonPath =
+      type === 'rpm'
+        ? `${version.replace('-', '.')}/0.1`
+        : version
+
+    try {
+      const existingPackageDetails =
+        await request({
+          uri: `https://${apiToken}:@packagecloud.io/api/v1/repos/AtomEditor/${packageRepoName}/package/${type}/${distroName}/${distroVersion}/atom${releaseSuffix || ''}/${arch}/${versionJsonPath}.json`,
+          method: 'get'
+        })
+
+      if (existingPackageDetails && existingPackageDetails.destroy_url) {
+        console.log(`Deleting pre-existing package ${fileName} in ${packageRepoName}`)
+        await request({
+          uri: `https://${apiToken}:@packagecloud.io/${existingPackageDetails.destroy_url}`,
+          method: 'delete'
+        })
+      }
+    } catch (err) {
+      if (err.statusCode !== 404) {
+        console.log(`Error while checking for existing '${fileName}' v${version}:\n\n`, err)
+      }
+    }
+  }
+}

script/package-lock.json

@@ -8760,6 +8760,24 @@
         "uuid": "^3.1.0"
       }
     },
+    "request-promise-core": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz",
+      "integrity": "sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=",
+      "requires": {
+        "lodash": "^4.13.1"
+      }
+    },
+    "request-promise-native": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.5.tgz",
+      "integrity": "sha1-UoF3D2jgyXGeUWP9P6tIIhX0/aU=",
+      "requires": {
+        "request-promise-core": "1.1.1",
+        "stealthy-require": "^1.1.0",
+        "tough-cookie": ">=2.3.3"
+      }
+    },
     "require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -9310,6 +9328,11 @@
         }
       }
     },
+    "stealthy-require": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
+      "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
+    },
     "string-editor": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/string-editor/-/string-editor-0.1.2.tgz",

script/package.json

@@ -30,6 +30,8 @@
     "pegjs": "0.9.0",
     "publish-release": "^1.6.0",
     "random-seed": "^0.3.0",
+    "request": "^2.87.0",
+    "request-promise-native": "^1.0.5",
     "season": "5.3.0",
     "semver": "5.3.0",
     "standard": "8.4.0",

script/publish-release

@@ -6,6 +6,8 @@ const path = require('path')
 const glob = require('glob')
 const publishRelease = require('publish-release')
 const uploadToS3 = require('./lib/upload-to-s3')
+const uploadLinuxPackages = require('./lib/upload-linux-packages')
+
 const CONFIG = require('./config')
 
 const yargs = require('yargs')
@@ -15,6 +17,7 @@ const argv = yargs
   .describe('assets-path', 'Path to the folder where all release assets are stored')
   .describe('s3-path', 'Indicates the S3 path in which the assets should be uploaded')
   .describe('create-github-release', 'Creates a GitHub release for this build, draft if release branch or public if Nightly')
+  .describe('linux-repo-name', 'If specified, uploads Linux packages to the given repo name on packagecloud')
   .wrap(yargs.terminalWidth())
   .argv
 
@@ -28,38 +31,61 @@ if (!assets || assets.length === 0) {
   process.exit(1)
 }
 
-console.log(`Uploading ${assets.length} release assets for ${CONFIG.computedAppVersion} to S3 under '${bucketPath}'`)
+async function uploadRelease() {
+  console.log(`Uploading ${assets.length} release assets for ${CONFIG.computedAppVersion} to S3 under '${bucketPath}'`)
 
-uploadToS3(
-  process.env.ATOM_RELEASES_S3_KEY,
-  process.env.ATOM_RELEASES_S3_SECRET,
-  process.env.ATOM_RELEASES_S3_BUCKET,
-  bucketPath,
-  assets).then(
-    () => {
-      if (argv.createGithubRelease) {
-        console.log(`Creating GitHub release v${CONFIG.computedAppVersion}`)
-        publishRelease({
-          token: process.env.GITHUB_TOKEN,
-          owner: 'atom',
-          repo: CONFIG.channel !== 'nightly' ? 'atom' : 'atom-nightly-releases',
-          name: CONFIG.computedAppVersion,
-          tag: `v${CONFIG.computedAppVersion}`,
-          draft: false,
-          prerelease: CONFIG.channel !== 'stable',
-          reuseRelease: true,
-          skipIfPublished: true,
-          assets
-        }, function (err, release) {
-          if (err) {
-            console.error("An error occurred while publishing the release:\n\n", err)
-          } else {
-            console.log("Release published successfully: ", release.html_url)
-          }
-        })
+  await uploadToS3(
+    process.env.ATOM_RELEASES_S3_KEY,
+    process.env.ATOM_RELEASES_S3_SECRET,
+    process.env.ATOM_RELEASES_S3_BUCKET,
+    bucketPath,
+    assets)
+
+  if (argv.linuxRepoName) {
+    await uploadLinuxPackages(
+      argv.linuxRepoName,
+      process.env.PACKAGE_CLOUD_API_KEY,
+      CONFIG.computedAppVersion,
+      assets)
+  } else {
+      console.log("Skipping upload of Linux packages")
+  }
+
+  if (argv.createGithubRelease) {
+    console.log(`Creating GitHub release v${CONFIG.computedAppVersion}`)
+    const release =
+      await publishReleaseAsync({
+        token: process.env.GITHUB_TOKEN,
+        owner: 'atom',
+        repo: CONFIG.channel !== 'nightly' ? 'atom' : 'atom-nightly-releases',
+        name: CONFIG.computedAppVersion,
+        tag: `v${CONFIG.computedAppVersion}`,
+        draft: false,
+        prerelease: CONFIG.channel !== 'stable',
+        reuseRelease: true,
+        skipIfPublished: true,
+        assets
+      })
+
+    console.log("Release published successfully: ", release.html_url)
+  } else {
+    console.log("Skipping GitHub release creation")
+  }
+}
+
+async function publishReleaseAsync(options) {
+  return new Promise((resolve, reject) => {
+    publishRelease(options, (err, release) => {
+      if (err) {
+        reject(err)
       } else {
-        console.log("Skipping GitHub release creation")
+        resolve(release)
       }
-    }).catch((err) => {
-      console.error('An error occurred while uploading the release:', err)
     })
+  })
+}
+
+uploadRelease().catch((err) => {
+  console.error('An error occurred while uploading the release:\n\n', err)
+  process.exit(1)
+})

script/vsts/nightly-release.yml

@@ -61,4 +61,5 @@ jobs:
       ATOM_RELEASES_S3_KEY: $(ATOM_RELEASES_S3_KEY)
       ATOM_RELEASES_S3_SECRET: $(ATOM_RELEASES_S3_SECRET)
       ATOM_RELEASES_S3_BUCKET: $(ATOM_RELEASES_S3_BUCKET)
+      PACKAGE_CLOUD_API_KEY: $(PACKAGE_CLOUD_API_KEY)
     displayName: Create Nightly Release

script/vsts/release-branch-build.yml

@@ -66,5 +66,6 @@ jobs:
       ATOM_RELEASES_S3_KEY: $(ATOM_RELEASES_S3_KEY)
       ATOM_RELEASES_S3_SECRET: $(ATOM_RELEASES_S3_SECRET)
       ATOM_RELEASES_S3_BUCKET: $(ATOM_RELEASES_S3_BUCKET)
+      PACKAGE_CLOUD_API_KEY: $(PACKAGE_CLOUD_API_KEY)
     displayName: Upload CI Artifacts to S3
     condition: and(succeeded(), eq(variables['IsSignedZipBranch'], 'true'))