playwright/utils/linux-browser-dependencies/inside_docker/list_dependencies.js
Andrey Lushnikov d499cf08d1
refactor: remove browserPaths in favor of Registry class (#5318)
This patch introduces a new Registry class that incapsulates
all logic regarding browsers and their paths.

Fixes #5278
2021-02-08 16:02:49 -08:00

286 lines
10 KiB
JavaScript

#!/usr/bin/env node
const fs = require('fs');
const util = require('util');
const path = require('path');
const {spawn} = require('child_process');
const {registryDirectory} = require('playwright/lib/utils/registry.js');
const readdirAsync = util.promisify(fs.readdir.bind(fs));
const readFileAsync = util.promisify(fs.readFile.bind(fs));
const readline = require('readline');
// These libraries are accessed dynamically by browsers using `dlopen` system call and
// thus have to be installed in the system.
//
// Tip: to assess which libraries are getting opened dynamically, one can use `strace`:
//
// strace -f -e trace=open,openat <program>
//
const DL_OPEN_LIBRARIES = {
chromium: [],
firefox: [],
webkit: [ 'libGLESv2.so.2' ],
};
(async () => {
console.log('Working on:', await getDistributionName());
console.log('Started at:', currentTime());
const browserDescriptors = (await readdirAsync(registryDirectory)).filter(dir => !dir.startsWith('.')).map(dir => ({
// Full browser name, e.g. `webkit-1144`
name: dir,
// Full patch to browser files
path: path.join(registryDirectory, dir),
// All files that we will try to inspect for missing dependencies.
filePaths: [],
// All libraries that are missing for the browser.
missingLibraries: new Set(),
// All packages required for the browser.
requiredPackages: new Set(),
// Libraries that we didn't find a package.
unresolvedLibraries: new Set(),
}));
// Collect all missing libraries for all browsers.
const allMissingLibraries = new Set();
for (const descriptor of browserDescriptors) {
// Browser vendor, can be `webkit`, `firefox` or `chromium`
const vendor = descriptor.name.split('-')[0];
for (const library of DL_OPEN_LIBRARIES[vendor]) {
descriptor.missingLibraries.add(library);
allMissingLibraries.add(library);
}
const {stdout} = await runCommand('find', [descriptor.path, '-type', 'f']);
descriptor.filePaths = stdout.trim().split('\n').map(f => f.trim()).filter(filePath => !filePath.toLowerCase().endsWith('.sh'));
await Promise.all(descriptor.filePaths.map(async filePath => {
const missingLibraries = await missingFileDependencies(filePath);
for (const library of missingLibraries) {
descriptor.missingLibraries.add(library);
allMissingLibraries.add(library);
}
}));
}
const libraryToPackage = new Map();
const ambiguityLibraries = new Map();
// Map missing libraries to packages that could be installed to fulfill the dependency.
console.log(`Finding packages for ${allMissingLibraries.size} missing libraries...`);
for (let i = 0, array = [...allMissingLibraries].sort(); i < allMissingLibraries.size; ++i) {
const library = array[i];
const packages = await findPackages(library);
const progress = `${i + 1}/${allMissingLibraries.size}`;
console.log(`${progress.padStart(7)}: ${library} => ${JSON.stringify(packages)}`);
if (!packages.length) {
const browsersWithMissingLibrary = browserDescriptors.filter(d => d.missingLibraries.has(library)).map(d => d.name).join(', ');
const PADDING = ''.padStart(7) + ' ';
console.log(PADDING + `ERROR: failed to resolve '${library}' required by ${browsersWithMissingLibrary}`);
} else if (packages.length === 1) {
libraryToPackage.set(library, packages[0]);
} else {
ambiguityLibraries.set(library, packages);
}
}
console.log('');
console.log(`Picking packages for ${ambiguityLibraries.size} libraries that have multiple package candidates`);
// Pick packages to install to fulfill missing libraries.
//
// This is a 2-step process:
// 1. Pick easy libraries by filtering out debug, test and dev packages.
// 2. After that, pick packages that we already picked before.
// Step 1: pick libraries that are easy to pick.
const totalAmbiguityLibraries = ambiguityLibraries.size;
for (const [library, packages] of ambiguityLibraries) {
const package = pickPackage(library, packages);
if (!package)
continue;
libraryToPackage.set(library, package);
ambiguityLibraries.delete(library);
const progress = `${totalAmbiguityLibraries - ambiguityLibraries.size}/${totalAmbiguityLibraries}`;
console.log(`${progress.padStart(7)}: ${library} => ${package}`);
console.log(''.padStart(9) + `(note) packages are ${JSON.stringify(packages)}`);
}
// 2nd pass - prefer packages that we already picked.
const allUsedPackages = new Set(libraryToPackage.values());
for (const [library, packages] of ambiguityLibraries) {
const package = packages.find(package => allUsedPackages.has(package));
if (!package)
continue;
libraryToPackage.set(library, package);
ambiguityLibraries.delete(library);
const progress = `${totalAmbiguityLibraries - ambiguityLibraries.size}/${totalAmbiguityLibraries}`;
console.log(`${progress.padStart(7)}: ${library} => ${package}`);
console.log(''.padStart(9) + `(note) packages are ${JSON.stringify(packages)}`);
}
// 3rd pass - prompt user to resolve ambiguity.
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
const promptAsync = (question) => new Promise(resolve => rl.question(question, resolve));
// Report all ambiguities that were failed to resolve.
for (const [library, packages] of ambiguityLibraries) {
const question = [
`Pick a package for '${library}':`,
...packages.map((package, index) => ` (${index + 1}) ${package}`),
'Enter number: ',
].join('\n');
const answer = await promptAsync(question);
const index = parseInt(answer, 10) - 1;
if (isNaN(index) || (index < 0) || (index >= packages.length)) {
console.error(`ERROR: unknown index "${answer}". Must be a number between 1 and ${packages.length}`);
process.exit(1);
}
const package = packages[index];
ambiguityLibraries.delete(library);
libraryToPackage.set(library, package);
console.log(answer);
console.log(`- ${library} => ${package}`);
}
rl.close();
// For each browser build a list of packages to install.
for (const descriptor of browserDescriptors) {
for (const library of descriptor.missingLibraries) {
const package = libraryToPackage.get(library);
if (package)
descriptor.requiredPackages.add(package);
else
descriptor.unresolvedLibraries.add(library);
}
}
// Formatting results.
console.log('');
console.log(`----- Library to package name mapping -----`);
console.log('{');
const sortedEntries = [...libraryToPackage.entries()].sort((a, b) => a[0].localeCompare(b[0]));
for (const [library, package] of sortedEntries)
console.log(` "${library}": "${package}",`);
console.log('}');
// Packages and unresolved libraries for every browser
for (const descriptor of browserDescriptors) {
console.log('');
console.log(`======= ${descriptor.name}: required packages =======`);
const requiredPackages = [...descriptor.requiredPackages].sort();
console.log(JSON.stringify(requiredPackages, null, 2));
console.log('');
console.log(`------- ${descriptor.name}: unresolved libraries -------`);
const unresolvedLibraries = [...descriptor.unresolvedLibraries].sort();
console.log(JSON.stringify(unresolvedLibraries, null, 2));
}
const status = browserDescriptors.some(d => d.unresolvedLibraries.size) ? 'FAILED' : 'SUCCESS';
console.log(`
====================
${status}
====================
`);
})();
function pickPackage(library, packages) {
// Step 1: try to filter out debug, test and dev packages.
packages = packages.filter(package => !package.endsWith('-dbg') && !package.endsWith('-test') && !package.endsWith('-dev') && !package.endsWith('-mesa'));
if (packages.length === 1)
return packages[0];
// Step 2: use library name to filter packages with the same name.
const prefix = library.split(/[-.]/).shift().toLowerCase();
packages = packages.filter(package => package.toLowerCase().startsWith(prefix));
if (packages.length === 1)
return packages[0];
return null;
}
async function findPackages(libraryName) {
const {stdout} = await runCommand('apt-file', ['search', libraryName]);
if (!stdout.trim())
return [];
const libs = stdout.trim().split('\n').map(line => line.split(':')[0]);
return [...new Set(libs)];
}
async function fileDependencies(filePath) {
const {stdout, code} = await lddAsync(filePath);
if (code !== 0)
return [];
const deps = stdout.split('\n').map(line => {
line = line.trim();
const missing = line.includes('not found');
const name = line.split('=>')[0].trim();
return {name, missing};
});
return deps;
}
async function missingFileDependencies(filePath) {
const deps = await fileDependencies(filePath);
return deps.filter(dep => dep.missing).map(dep => dep.name);
}
async function lddAsync(filePath) {
let LD_LIBRARY_PATH = [];
// Some shared objects inside browser sub-folders link against libraries that
// ship with the browser. We consider these to be included, so we want to account
// for them in the LD_LIBRARY_PATH.
for (let dirname = path.dirname(filePath); dirname !== '/'; dirname = path.dirname(dirname))
LD_LIBRARY_PATH.push(dirname);
return await runCommand('ldd', [filePath], {
cwd: path.dirname(filePath),
env: {
...process.env,
LD_LIBRARY_PATH: LD_LIBRARY_PATH.join(':'),
},
});
}
function runCommand(command, args, options = {}) {
const childProcess = spawn(command, args, options);
return new Promise((resolve) => {
let stdout = '';
let stderr = '';
childProcess.stdout.on('data', data => stdout += data);
childProcess.stderr.on('data', data => stderr += data);
childProcess.on('close', (code) => {
resolve({stdout, stderr, code});
});
});
}
async function getDistributionName() {
const osReleaseText = await readFileAsync('/etc/os-release', 'utf8');
const fields = new Map();
for (const line of osReleaseText.split('\n')) {
const tokens = line.split('=');
const name = tokens.shift();
let value = tokens.join('=').trim();
if (value.startsWith('"') && value.endsWith('"'))
value = value.substring(1, value.length - 1);
if (!name)
continue;
fields.set(name.toLowerCase(), value);
}
return fields.get('pretty_name') || '';
}
function currentTime() {
const date = new Date();
const dateTimeFormat = new Intl.DateTimeFormat('en', { year: 'numeric', month: 'short', day: '2-digit' });
const [{ value: month },,{ value: day },,{ value: year }] = dateTimeFormat .formatToParts(date );
return `${month} ${day}, ${year}`;
}