F1bonacc1/process-compose
1
1
Fork 0
mirror of https://github.com/F1bonacc1/process-compose.git synced 2024-10-03 22:27:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
Berger Eugene 2022-04-06 00:26:50 +03:00
commit 03458d14d1
28 changed files with 1922 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
coverage.out
.vscode
bin/**
*.log
.env

201
LICENSE Normal file
View File

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2022 Berger Eugene
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

24
Makefile Normal file
View File

@ -0,0 +1,24 @@
BINARY_NAME=process-compose
.PHONY: test run
buildrun: build run
build:
go build -o bin/${BINARY_NAME} ./src
compile:
GOOS=linux GOARCH=386 go build -o bin/${BINARY_NAME}-linux-386 ./src
GOOS=linux GOARCH=amd64 go build -o bin/${BINARY_NAME}-linux-amd64 ./src
GOOS=linux GOARCH=arm64 go build -o bin/${BINARY_NAME}-linux-arm64 ./src
GOOS=linux GOARCH=arm go build -o bin/${BINARY_NAME}-linux-arm ./src
test:
go test -cover ./src
coverhtml:
go test -coverprofile=coverage.out ./src
go tool cover -html=coverage.out
run:
./bin/${BINARY_NAME}
clean:
rm bin/${BINARY_NAME}*

197
README.md Executable file
View File

@ -0,0 +1,197 @@
## Process Compose
[![made-with-Go](https://img.shields.io/badge/Made%20with-Go-1f425f.svg)](https://go.dev/) [![Linux](https://svgshare.com/i/Zhy.svg)](https://svgshare.com/i/Zhy.svg) [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/Naereen/StrapDown.js/graphs/commit-activity) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com)
Process compose is a small utility for building custom workflows and execution sequences. It is optimized for:
* Parallelizing processes execution
* Defining execution dependencies and order
* Defining recovery policies (restart `on-failure`, `always`, `no`)
* Declaring processes arguments
* Declaring processes environment variables
It is heavily inspired by [docker-compose](https://github.com/docker/compose), but without the need for containers. The configuration syntax tries to follow the docker-compose specifications, with a few minor additions and lots of subtractions.
### Installation
- Download one of the [releases](https://github.com/F1bonacc1/process-compose/releases)
### Documentation
* See [examples](https://github.com/F1bonacc1/process-compose/tree/main/examples) of workflows for best practices
* See below
#### List of Features and Planned Features
##### ✅ Mostly implemented
##### ❌ Implementation not started (Your feedback and ⭐ will motivate further development 😃)
#### ✅ <u>Launcher</u>
##### ✅ Parallel
```yaml
process1:
command: "sleep 3"
process2:
command: "sleep 3"
```
##### ✅ Serial
```yaml
process1:
command: "sleep 3"
depends_on:
process2:
condition: process_completed_successfully # or "process_completed" if you don't care about errors
process2:
command: "sleep 3"
depends_on:
process3:
condition: process_completed_successfully # or "process_completed" if you don't care about errors
```
##### ❌ Instance Number
##### ✅ Define process dependencies
```yaml
process2:
depends_on:
process2:
condition: process_completed_successfully # or "process_started" (default)
process3:
condition: process_completed_successfully
```
#### ✅ <u>Output Handling</u>
##### ✅ Show process name
##### ✅ Different colors per process
##### ✅ StdErr is printed in Red
<img src="./imgs/output.png" alt="output" style="zoom:50%;" />
##### ❌ Silence specific processes
#### ✅ <u>Logger</u>
##### ✅ Per Process Log Collection
```yaml
process2:
log_location: ./pc.process2.log #if undefined or empty no logs will be saved
```
##### ✅ Capture StdOut output
##### ✅ Capture StdErr output
##### ✅ Merge into a single file
```yaml
processes:
process2:
command: "chmod 666 /path/to/file"
environment:
- 'ABC=42'
log_location: ./pc.global.log #if undefined or empty no logs will be saved (if also not defined per process)
```
##### ❌ Silence specific processes
#### ❌ <u>Health Checks</u>
##### ❌ Is Alive
##### ❌ Is Ready
##### ❌ Auto Restart if not healthy
##### ✅ Auto Restart on exit
```yaml
process2:
availability:
restart: on-failure # other options: "always", "no" (default)
backoff_seconds: 2 # default: 1
max_restarts: 5 # default: 0 (unlimited)
```
#### ✅ <u>Environment Variables</u>
##### ✅ Per Process
```yaml
process2:
environment:
- 'I_AM_LOCAL_EV=42'
```
##### ✅ Global
```yaml
processes:
process2:
command: "chmod 666 /path/to/file"
environment:
- 'I_AM_LOCAL_EV=42'
environment:
- 'I_AM_GLOBAL_EV=42'
```
#### ❌ <u>System Variables</u>
##### ❌ Process replica number
##### ❌ Monitoring
##### ❌ REST API
#### ✅ <u>Configuration</u>
##### ✅ Support .env file
##### ✅ Override ${var} and $var from environment variables or .env values
##### ❌ Merge 2 or more configuration files with override values
##### ✅ Specify which configuration files to use
```shell
process-compose -f "path/to/process-compose-file.yaml"
```
##### ✅ Auto discover configuration files
The following discovery order is used: `compose.yml, compose.yaml, process-compose.yml, process-compose.yaml`. If multiple files are present the first one will be used.
#### ❌ <u>Multi-platform</u>
##### ✅ Linux
##### ❌ Windows
##### ❌ macOS

4
examples/.example_env Normal file
View File

@ -0,0 +1,4 @@
VERSION='1.2.3'
DB_USER='USERNAME'
DB_PASSWORD='VERY_STRONG_PASSWORD'
WAIT_SEC=60

80
examples/data_aggregation_workflow.yaml Normal file
View File

@ -0,0 +1,80 @@
version: "0.5"
environment:
- 'IS_PRODUCTION_MODE=1'
log_location: ./pc.log
processes:
# downloader_A runs every 5min to download new data from its specific uri
# the download directory is also specific for this process
# the version of the downloader is defined in the .example_env (because DRY)
downloader_A:
command: "python3 data_downloader_${VERSION}.py -s 'data.source.A.uri'"
availability:
restart: "always"
backoff_seconds: 300
environment:
- 'OUTPUT_DIR=/path/to/A/data'
# downloader_B also runs every 5min to download new data from its specific uri
# the download directory is also specific for this process
# the version of the downloader is defined in the .env (because DRY)
downloader_B:
command: "python3 data_downloader_${VERSION}.py -s 'data.source.B.uri'"
availability:
restart: "always"
backoff_seconds: 300
environment:
- 'OUTPUT_DIR=/path/to/B/data'
# cleanser should be always running.
# It will restart in case of failure with a 2 seconds delay
# It cleans the data downloaded by the downloaders and removes duplicates
cleanser:
command: "python3 data_cleanser.py --path /path/to/A/data --path /path/to/B/data"
availability:
restart: "on-failure"
backoff_seconds: 2
environment:
- 'OUTPUT_DIR=/path/to/cleansed/data'
# db_feeder should be always running.
# It will restart in case of failure with a 1 second (default) delay
# It feeds the clean data into the DB
# It depends on the db_awaiter and won't run until it exits with status 0
# The DB_USER and DB_PASSWORD are defined in the .env file (should be in .gitignore)
db_feeder:
command: "python3 feed_db.py --source /path/to/cleansed/data"
availability:
restart: on-failure
depends_on:
db_awaiter:
condition: process_completed_successfully
environment:
- 'DB_CONNECTION_STRING=MYDB://${DB_USER}:${DB_PASSWORD}@localhost:27017'
# db_awaiter is designed to run once and exit
# It sleeps for ${WAIT_SEC} seconds (defined in the .env file) and exits
# It runs imideately after the DB is started
# This allows the DB to start before processes which depend on it try to establish connection
# multiple processes can declare that they depend on db_awaiter
db_awaiter:
command: "echo 'Waiting for ${WAIT_SEC} seconds for DB to start... && sleep ${WAIT_SEC}"
availability:
restart: "no"
depends_on:
database:
condition: process_started
# database should be always running
# It uses an external script for spinning up the DB (or maybe it just hides a docker command...)
# Its log is saved to a sepparate file since this DB is too chatty
database:
command: "/path/to/db_runner.sh"
availability:
restart: always
log_location: "/path/to/db.log"
environment:
- 'DB_SPECIFIC_VAR1=VAL1'
- 'DB_SPECIFIC_VAR2=VAL2'

71
fixtures/process-compose-chain-arrow.yaml Normal file
View File

@ -0,0 +1,71 @@
version: "0.5"
processes:
process1:
command: "echo process1"
depends_on:
process2:
condition: process_completed_successfully
process2:
command: "echo process2"
availability:
restart: "on-failure"
depends_on:
process3:
condition: process_completed_successfully
process3:
command: "echo process3"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process4:
condition: process_completed_successfully
process4:
command: "echo process4"
availability:
restart: "on-failure"
depends_on:
process5:
condition: process_completed_successfully
process5:
command: "echo process5"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process6:
condition: process_completed_successfully
process6:
command: "echo process6"
availability:
restart: "on-failure"
depends_on:
process7:
condition: process_completed_successfully
process8:
condition: process_completed_successfully
process7:
command: "echo process7"
availability:
restart: "on-failure"
depends_on:
process8:
condition: process_completed_successfully
process8:
command: "echo process8"
availability:
restart: "on-failure"
backoff_seconds: 2
environment:
- 'ABC=222'

71
fixtures/process-compose-chain-rhombus.yaml Normal file
View File

@ -0,0 +1,71 @@
version: "0.5"
processes:
process1:
command: "echo process1"
depends_on:
process2:
condition: process_completed_successfully
process2:
command: "echo process2"
availability:
restart: "on-failure"
depends_on:
process3:
condition: process_completed_successfully
process3:
command: "echo process3"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process4:
condition: process_completed_successfully
process4:
command: "echo process4"
availability:
restart: "on-failure"
depends_on:
process5:
condition: process_completed_successfully
process5:
command: "echo process5"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process6:
condition: process_completed_successfully
process7:
condition: process_completed_successfully
process6:
command: "echo process6"
availability:
restart: "on-failure"
depends_on:
process8:
condition: process_completed_successfully
process7:
command: "echo process7"
availability:
restart: "on-failure"
depends_on:
process8:
condition: process_completed_successfully
process8:
command: "echo process8"
availability:
restart: "on-failure"
backoff_seconds: 2
environment:
- 'ABC=222'

69
fixtures/process-compose-chain-with-errors.yaml Normal file
View File

@ -0,0 +1,69 @@
version: "0.5"
processes:
process1:
command: "echo process1"
depends_on:
process2:
condition: process_completed_successfully
process2:
command: "echo process2"
availability:
restart: "on-failure"
depends_on:
process3:
condition: process_completed_successfully
process3:
command: "echo 'process3 error' >&2 && exit 1"
depends_on:
process4:
condition: process_completed_successfully
process4:
command: "echo process4"
availability:
restart: "on-failure"
depends_on:
process5:
condition: process_completed
process5:
command: "echo 'process5 error' >&2 && exit 1"
availability:
restart: "on-failure"
backoff_seconds: 1
max_restarts: 1
depends_on:
process6:
condition: process_completed_successfully
process6:
command: "echo process6"
availability:
restart: "always"
backoff_seconds: 2
max_restarts: 1
depends_on:
process7:
condition: process_completed_successfully
process7:
command: "echo process7"
availability:
restart: "on-failure"
depends_on:
process8:
condition: process_completed_successfully
process8:
command: "echo process8"
availability:
restart: "on-failure"
backoff_seconds: 2
environment:
- 'ABC=222'

69
fixtures/process-compose-chain.yaml Normal file
View File

@ -0,0 +1,69 @@
version: "0.5"
processes:
process1:
command: "echo process1"
depends_on:
process2:
condition: process_completed_successfully
process2:
command: "echo process2"
availability:
restart: "on-failure"
depends_on:
process3:
condition: process_completed_successfully
process3:
command: "echo process3"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process4:
condition: process_completed_successfully
process4:
command: "echo process4"
availability:
restart: "on-failure"
depends_on:
process5:
condition: process_completed_successfully
process5:
command: "echo process5"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process6:
condition: process_completed_successfully
process6:
command: "echo process6"
availability:
restart: "on-failure"
depends_on:
process7:
condition: process_completed_successfully
process7:
command: "echo process7"
availability:
restart: "on-failure"
depends_on:
process8:
condition: process_completed_successfully
process8:
command: "echo process8"
availability:
restart: "on-failure"
backoff_seconds: 2
environment:
- 'ABC=222'

70
fixtures/process-compose-many-for-one.yaml Normal file
View File

@ -0,0 +1,70 @@
version: "0.5"
processes:
process2:
command: "echo process2"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process1:
condition: process_completed_successfully
process1:
command: "echo process1"
availability:
restart: "on-failure"
process4:
command: "echo process4"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process1:
condition: process_completed_successfully
process3:
command: "echo process3"
availability:
restart: "on-failure"
depends_on:
process1:
condition: process_completed_successfully
process6:
command: "echo process6"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process1:
condition: process_completed_successfully
process5:
command: "echo process5"
availability:
restart: "on-failure"
depends_on:
process1:
condition: process_completed_successfully
process8:
command: "echo process8"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process1:
condition: process_completed_successfully
process7:
command: "echo process7"
availability:
restart: "on-failure"
depends_on:
process1:
condition: process_completed_successfully
environment:
- 'ABC=222'

61
fixtures/process-compose-odd-then-even.yaml Normal file
View File

@ -0,0 +1,61 @@
version: "0.5"
processes:
process2:
command: "echo process2"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process1:
condition: process_completed_successfully
process1:
command: "echo process1"
availability:
restart: "on-failure"
process4:
command: "echo process4"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process3:
condition: process_completed_successfully
process3:
command: "echo process3"
availability:
restart: "on-failure"
process6:
command: "echo process6"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process5:
condition: process_completed_successfully
process5:
command: "echo process5"
availability:
restart: "on-failure"
process8:
command: "echo process8"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process7:
condition: process_completed_successfully
process7:
command: "echo process7"
availability:
restart: "on-failure"
environment:
- 'ABC=222'

51
fixtures/process-compose-one-for-many.yaml Normal file
View File

@ -0,0 +1,51 @@
version: "0.5"
processes:
process2:
command: "echo process2"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process1:
condition: process_completed_successfully
process3:
condition: process_completed_successfully
process4:
condition: process_completed_successfully
process5:
condition: process_completed_successfully
process6:
condition: process_completed_successfully
process7:
condition: process_completed_successfully
process8:
condition: process_completed_successfully
process1:
command: "echo process1"
process4:
command: "echo process4"
process3:
command: "echo process3"
process6:
command: "echo process6"
process5:
command: "echo process5"
process8:
command: "echo process8"
process7:
command: "echo process7"
environment:
- 'ABC=222'

68
fixtures/process-compose-with-log.yaml Normal file
View File

@ -0,0 +1,68 @@
version: "0.5"
processes:
process1:
command: "echo process1"
depends_on:
process2:
condition: process_completed_successfully
process2:
command: "echo 'process2 is removing the log' && rm ./pc.log-test.log"
availability:
restart: "on-failure"
depends_on:
process3:
condition: process_completed
process3:
command: "echo 'process3 error' >&2 && exit 1"
depends_on:
process4:
condition: process_completed_successfully
process4:
command: "echo process4"
availability:
restart: "on-failure"
depends_on:
process5:
condition: process_completed_successfully
process5:
command: "echo 'process5 is removing the process6 log' && rm ./pc.proc6.log-test.log"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process6:
condition: process_completed_successfully
process6:
command: "echo process6"
availability:
restart: "on-failure"
log_location: ./pc.proc6.log-test.log
depends_on:
process7:
condition: process_completed_successfully
process7:
command: "echo process7"
availability:
restart: "on-failure"
depends_on:
process8:
condition: process_completed_successfully
process8:
command: "echo process8"
availability:
restart: "on-failure"
backoff_seconds: 2
environment:
- 'ABC=222'
log_location: ./pc.log-test.log

21
go.mod Normal file
View File

@ -0,0 +1,21 @@
module github.com/f1bonacc1/process-compose
go 1.18
require (
github.com/fatih/color v1.13.0
github.com/joho/godotenv v1.4.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)
require (
github.com/kr/pretty v0.1.0 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
)
require (
github.com/mattn/go-colorable v0.1.9 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/rs/zerolog v1.26.1
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect
)

55
go.sum Normal file
View File

@ -0,0 +1,55 @@
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U=
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc=
github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

BIN
imgs/output.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 238 KiB

57
process-compose.yaml Normal file
View File

@ -0,0 +1,57 @@
version: "0.5"
processes:
process0:
command: "ls ddd"
process1:
command: "./test_loop.bash ${PROC4}"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process2:
condition: process_completed_successfully
process3:
condition: process_completed
# process4:
# condition: process_completed_successfully
environment:
- 'EXIT_CODE=0'
process2:
command: "./test_loop.bash process2"
log_location: ./pc.proc2.log
availability:
restart: "on-failure"
# depends_on:
# process3:
# condition: process_completed_successfully
environment:
- 'ABC=2221'
- 'PRINT_ERR=111'
- 'EXIT_CODE=0'
process3:
command: "./test_loop.bash process3"
availability:
restart: "on-failure"
backoff_seconds: 2
depends_on:
process4:
condition: process_completed_successfully
process4:
command: "./test_loop.bash process4"
# availability:
# restart: on-failure
environment:
- 'ABC=2221'
- 'EXIT_CODE=1'
kcalc:
command: "kcalc"
disabled: true
environment:
- 'ABC=222'
log_location: ./pc.log

69
src/config.go Normal file
View File

@ -0,0 +1,69 @@
package main
import "sync"
type Project struct {
Version string `yaml:"version"`
LogLocation string `yaml:"log_location,omitempty"`
Processes Processes `yaml:"processes"`
Environment []string `yaml:"environment,omitempty"`
runningProcesses map[string]*Process
mapMutex sync.Mutex
}
type Processes map[string]ProcessConfig
type ProcessConfig struct {
Name string
Disabled bool `yaml:"disabled,omitempty"`
Command string `yaml:"command"`
LogLocation string `yaml:"log_location,omitempty"`
Environment []string `yaml:"environment,omitempty"`
RestartPolicy RestartPolicyConfig `yaml:"availability,omitempty"`
DependsOn DependsOnConfig `yaml:"depends_on,omitempty"`
Extensions map[string]interface{} `yaml:",inline"`
}
func (p ProcessConfig) GetDependencies() []string {
dependencies := make([]string, len(p.DependsOn))
i := 0
for k := range p.DependsOn {
dependencies[i] = k
i++
}
return dependencies
}
const (
RestartPolicyAlways = "always"
RestartPolicyOnFailure = "on-failure"
RestartPolicyNo = "no"
)
type RestartPolicyConfig struct {
Restart string `yaml:",omitempty"`
BackoffSeconds int `yaml:"backoff_seconds,omitempty"`
MaxRestarts int `yaml:"max_restarts,omitempty"`
}
const (
// ProcessConditionCompleted is the type for waiting until a process has completed (any exit code).
ProcessConditionCompleted = "process_completed"
// ProcessConditionCompletedSuccessfully is the type for waiting until a process has completed successfully (exit code 0).
ProcessConditionCompletedSuccessfully = "process_completed_successfully"
// ProcessConditionHealthy is the type for waiting until a process is healthy.
ProcessConditionHealthy = "process_healthy"
// ProcessConditionStarted is the type for waiting until a process has started (default).
ProcessConditionStarted = "process_started"
)
type DependsOnConfig map[string]ProcessDependency
type ProcessDependency struct {
Condition string `yaml:",omitempty"`
Extensions map[string]interface{} `yaml:",inline"`
}

36
src/logger_facade.go Normal file
View File

@ -0,0 +1,36 @@
package main
import (
"os"
"github.com/rs/zerolog"
)
type PCLog struct {
logger zerolog.Logger
}
func NewLogger(outputPath string) *PCLog {
f, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
if err != nil {
panic(err)
}
return &PCLog{
logger: zerolog.New(f),
}
}
func (l PCLog) Info(message string, process string, replica int) {
l.logger.Info().
Str("process", process).
Int("replica", replica).
Msg(message)
}
func (l PCLog) Error(message string, process string, replica int) {
l.logger.Error().
Str("process", process).
Int("replica", replica).
Msg(message)
}

6
src/logger_interface.go Normal file
View File

@ -0,0 +1,6 @@
package main
type PcLogger interface {
Info(message string, process string, replica int)
Error(message string, process string, replica int)
}

108
src/main.go Normal file
View File

@ -0,0 +1,108 @@
package main
import (
"errors"
"flag"
"fmt"
"io/ioutil"
"path/filepath"
"strings"
//"log"
"os"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/joho/godotenv"
"gopkg.in/yaml.v3"
)
func createProject(inputFile string) *Project {
yamlFile, err := ioutil.ReadFile(inputFile)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
log.Error().Msgf("File %s doesn't exist", inputFile)
}
log.Fatal().Msg(err.Error())
}
// .env is optional we don't care if it errors
godotenv.Load()
yamlFile = []byte(os.ExpandEnv(string(yamlFile)))
var project Project
err = yaml.Unmarshal(yamlFile, &project)
if err != nil {
log.Fatal().Msg(err.Error())
}
return &project
}
func setupLogger() {
log.Logger = log.Output(zerolog.ConsoleWriter{
Out: os.Stdout,
TimeFormat: "06-01-02 15:04:05",
})
}
func findFiles(names []string, pwd string) []string {
candidates := []string{}
for _, n := range names {
f := filepath.Join(pwd, n)
if _, err := os.Stat(f); err == nil {
candidates = append(candidates, f)
}
}
return candidates
}
// DefaultFileNames defines the Compose file names for auto-discovery (in order of preference)
var DefaultFileNames = []string{"compose.yml", "compose.yaml", "process-compose.yml", "process-compose.yaml"}
func autoDiscoverComposeFile(pwd string) (string, error) {
candidates := findFiles(DefaultFileNames, pwd)
if len(candidates) > 0 {
winner := candidates[0]
if len(candidates) > 1 {
log.Warn().Msgf("Found multiple config files with supported names: %s", strings.Join(candidates, ", "))
log.Warn().Msgf("Using %s", winner)
}
return winner, nil
}
return "", fmt.Errorf("no config files found in %s", pwd)
}
func isFlagPassed(name string) bool {
found := false
flag.Visit(func(f *flag.Flag) {
if f.Name == name {
found = true
}
})
return found
}
func main() {
setupLogger()
fileName := ""
flag.StringVar(&fileName, "f", DefaultFileNames[0], "path to file to load")
flag.Parse()
if !isFlagPassed("f") {
pwd, err := os.Getwd()
if err != nil {
log.Fatal().Msg(err.Error())
}
file, err := autoDiscoverComposeFile(pwd)
if err != nil {
log.Fatal().Msg(err.Error())
}
fileName = file
}
project := createProject(fileName)
project.Run()
}

20
src/nil_logger.go Normal file
View File

@ -0,0 +1,20 @@
package main
type PcNilLog struct {
}
func NewNilLogger(outputPath string) *PcNilLog {
return &PcNilLog{}
}
func (l *PcNilLog) Sync() {
}
func (l PcNilLog) Info(message string, process string, replica int) {
}
func (l PcNilLog) Error(message string, process string, replica int) {
}

179
src/process.go Normal file
View File

@ -0,0 +1,179 @@
package main
import (
"bufio"
"fmt"
"io"
"math/rand"
"os"
"os/exec"
"sync"
"time"
"github.com/fatih/color"
"github.com/rs/zerolog/log"
)
type Process struct {
globalEnv []string
procConf ProcessConfig
restartsCounter int
sync.Mutex
procCond sync.Cond
exitCode int
procColor func(a ...interface{}) string
noColor func(a ...interface{}) string
redColor func(a ...interface{}) string
logger PcLogger
done bool
replica int
}
func NewProcess(globalEnv []string, logger PcLogger, procConf ProcessConfig, replica int) *Process {
colNumeric := rand.Intn(int(color.FgHiWhite)-int(color.FgHiBlack)) + int(color.FgHiBlack)
//logger, _ := zap.NewProduction()
proc := &Process{
globalEnv: globalEnv,
procConf: procConf,
procColor: color.New(color.Attribute(colNumeric), color.Bold).SprintFunc(),
redColor: color.New(color.FgHiRed).SprintFunc(),
noColor: color.New(color.Reset).SprintFunc(),
logger: logger,
exitCode: -1,
done: false,
replica: replica,
}
proc.procCond = *sync.NewCond(proc)
return proc
}
func (p *Process) Run() error {
for {
cmd := exec.Command(getRunnerShell(), "-c", p.procConf.Command)
cmd.Env = p.getProcessEnvironment()
stdout, _ := cmd.StdoutPipe()
stderr, _ := cmd.StderrPipe()
go p.handleOutput(stdout, p.handleInfo)
go p.handleOutput(stderr, p.handleError)
cmd.Start()
cmd.Wait()
p.Lock()
p.exitCode = cmd.ProcessState.ExitCode()
p.Unlock()
log.Info().Msgf("%s exited with status %d", p.procConf.Name, p.exitCode)
if !p.isRestartable(p.exitCode) {
break
}
p.restartsCounter += 1
log.Info().Msgf("Restarting %s in %v second(s)... Restarts: %d",
p.procConf.Name, p.getBackoff().Seconds(), p.restartsCounter)
time.Sleep(p.getBackoff())
}
p.onProcessEnd()
return nil
}
func (p *Process) getBackoff() time.Duration {
backoff := 1
if p.procConf.RestartPolicy.BackoffSeconds > backoff {
backoff = p.procConf.RestartPolicy.BackoffSeconds
}
return time.Duration(backoff) * time.Second
}
func (p *Process) getProcessEnvironment() []string {
env := []string{
"PC_PROC_NAME=" + p.GetName(),
"PC_REPLICA_NUM=1",
}
env = append(env, os.Environ()...)
env = append(env, p.globalEnv...)
env = append(env, p.procConf.Environment...)
return env
}
func (p *Process) isRestartable(exitCode int) bool {
if p.procConf.RestartPolicy.Restart == RestartPolicyNo ||
p.procConf.RestartPolicy.Restart == "" {
return false
}
if exitCode != 0 && p.procConf.RestartPolicy.Restart == RestartPolicyOnFailure {
if p.procConf.RestartPolicy.MaxRestarts == 0 {
return true
}
return p.restartsCounter < p.procConf.RestartPolicy.MaxRestarts
}
if p.procConf.RestartPolicy.Restart == RestartPolicyAlways {
if p.procConf.RestartPolicy.MaxRestarts == 0 {
return true
}
return p.restartsCounter < p.procConf.RestartPolicy.MaxRestarts
}
return false
}
func (p *Process) WaitForCompletion(waitee string) int {
p.Lock()
defer p.Unlock()
for !p.done {
p.procCond.Wait()
}
return p.exitCode
}
func (p *Process) WontRun() {
p.onProcessEnd()
}
func (p *Process) onProcessEnd() {
p.Lock()
p.done = true
p.Unlock()
p.procCond.Broadcast()
}
func (p *Process) GetName() string {
return p.procConf.Name
}
func (p *Process) GetNameWithReplica() string {
return fmt.Sprintf("%s_%d", p.procConf.Name, p.replica)
}
func (p *Process) handleOutput(pipe io.ReadCloser,
handler func(message string)) {
outscanner := bufio.NewScanner(pipe)
outscanner.Split(bufio.ScanLines)
for outscanner.Scan() {
handler(outscanner.Text())
}
}
func (p *Process) handleInfo(message string) {
p.logger.Info(message, p.GetName(), p.replica)
fmt.Printf("[%s]\t%s\n", p.procColor(p.GetNameWithReplica()), p.noColor(message))
}
func (p *Process) handleError(message string) {
p.logger.Error(message, p.GetName(), p.replica)
fmt.Printf("[%s]\t%s\n", p.procColor(p.GetNameWithReplica()), p.redColor(message))
}
func getRunnerShell() string {
shell, ok := os.LookupEnv("SHELL")
if !ok {
return "bash"
} else {
return shell
}
}

159
src/project.go Normal file
View File

@ -0,0 +1,159 @@
package main
import (
"fmt"
"sync"
"github.com/rs/zerolog/log"
)
func (p *Project) Run() {
p.runningProcesses = make(map[string]*Process)
runOrder := []ProcessConfig{}
p.WithServices([]string{}, func(process ProcessConfig) error {
runOrder = append(runOrder, process)
return nil
})
var nameOrder []string
for _, v := range runOrder {
nameOrder = append(nameOrder, v.Name)
}
var logger PcLogger = NewNilLogger("")
if p.LogLocation != "" {
logger = NewLogger(p.LogLocation)
}
log.Debug().Msgf("Spinning up %d processes. Order: %q", len(runOrder), nameOrder)
var wg sync.WaitGroup
for _, proc := range runOrder {
procLogger := logger
if proc.LogLocation != "" {
procLogger = NewLogger(proc.LogLocation)
}
process := NewProcess(p.Environment, procLogger, proc, 1)
p.addRunningProcess(process)
wg.Add(1)
go func() {
defer p.removeRunningProcess(process.GetName())
defer wg.Done()
if err := p.WaitIfNeeded(process.procConf); err != nil {
log.Error().Msgf("Error: %s", err.Error())
log.Error().Msgf("Error: process %s won't run", process.GetName())
process.WontRun()
} else {
process.Run()
}
}()
}
wg.Wait()
}
func (p *Project) WaitIfNeeded(process ProcessConfig) error {
for k := range process.DependsOn {
if runningProc := p.getRunningProcess(k); runningProc != nil {
switch process.DependsOn[k].Condition {
case ProcessConditionCompleted:
runningProc.WaitForCompletion(process.Name)
case ProcessConditionCompletedSuccessfully:
log.Info().Msgf("%s is waiting for %s to complete successfully", process.Name, k)
exitCode := runningProc.WaitForCompletion(process.Name)
if exitCode != 0 {
return fmt.Errorf("process %s depended on %s to complete successfully, but it exited with status %d",
process.Name, k, exitCode)
}
}
}
}
return nil
}
func (p *Project) addRunningProcess(process *Process) {
p.mapMutex.Lock()
p.runningProcesses[process.GetName()] = process
p.mapMutex.Unlock()
}
func (p *Project) getRunningProcess(name string) *Process {
p.mapMutex.Lock()
defer p.mapMutex.Unlock()
if runningProc, ok := p.runningProcesses[name]; ok {
return runningProc
}
return nil
}
func (p *Project) removeRunningProcess(name string) {
p.mapMutex.Lock()
delete(p.runningProcesses, name)
p.mapMutex.Unlock()
}
func (p *Project) GetProcesses(names ...string) ([]ProcessConfig, error) {
processes := []ProcessConfig{}
if len(names) == 0 {
for name, proc := range p.Processes {
if proc.Disabled {
continue
}
proc.Name = name
processes = append(processes, proc)
}
return processes, nil
}
for _, name := range names {
if proc, ok := p.Processes[name]; ok {
if proc.Disabled {
continue
}
proc.Name = name
processes = append(processes, proc)
} else {
return processes, fmt.Errorf("no such process: %s", name)
}
}
return processes, nil
}
type ProcessFunc func(process ProcessConfig) error
// WithProcesses run ProcesseFunc on each Process and dependencies in dependency order
func (p *Project) WithServices(names []string, fn ProcessFunc) error {
return p.withProcesses(names, fn, map[string]bool{})
}
func (p *Project) withProcesses(names []string, fn ProcessFunc, done map[string]bool) error {
processes, err := p.GetProcesses(names...)
if err != nil {
return err
}
for _, process := range processes {
if done[process.Name] {
continue
}
done[process.Name] = true
dependencies := process.GetDependencies()
if len(dependencies) > 0 {
err := p.withProcesses(dependencies, fn, done)
if err != nil {
return err
}
}
if err := fn(process); err != nil {
return err
}
}
return nil
}
func (p *Project) GetDependenciesOrderNames() ([]string, error) {
order := []string{}
err := p.WithServices([]string{}, func(process ProcessConfig) error {
order = append(order, process.Name)
return nil
})
return order, err
}

132
src/project_test.go Normal file
View File

@ -0,0 +1,132 @@
package main
import (
"reflect"
"testing"
)
func TestProject_GetDependenciesOrderNames(t *testing.T) {
type fields struct {
Version string
LogLevel string
LogLocation string
Processes map[string]ProcessConfig
Environment []string
runningProcesses map[string]*Process
}
tests := []struct {
name string
fields fields
want []string
wantErr bool
}{
{
name: "ShouldBe_4321",
fields: fields{
Processes: map[string]ProcessConfig{
"Process1": {
Name: "Process1",
DependsOn: DependsOnConfig{
"Process2": {},
},
},
"Process2": {
Name: "Process2",
DependsOn: DependsOnConfig{
"Process3": {},
},
},
"Process3": {
Name: "Process3",
DependsOn: DependsOnConfig{
"Process4": {},
},
},
"Process4": {
Name: "Process4",
},
},
},
want: []string{"Process4", "Process3", "Process2", "Process1"},
wantErr: false,
},
{
name: "ShouldBe_Err",
fields: fields{
Processes: map[string]ProcessConfig{
"Process1": {
Name: "Process1",
DependsOn: DependsOnConfig{
"Process2": {},
},
},
"Process2": {
Name: "Process2",
DependsOn: DependsOnConfig{
"Process4": {},
},
},
},
},
want: []string{},
wantErr: true,
},
{
name: "ShouldBe_1",
fields: fields{
Processes: map[string]ProcessConfig{
"Process1": {
Name: "Process1",
DependsOn: DependsOnConfig{
"Process2": {},
},
},
"Process2": {
Name: "Process2",
Disabled: true,
},
},
},
want: []string{"Process1"},
wantErr: false,
},
{
name: "ShouldBe_2",
fields: fields{
Processes: map[string]ProcessConfig{
"Process1": {
Name: "Process1",
Disabled: true,
DependsOn: DependsOnConfig{
"Process2": {},
},
},
"Process2": {
Name: "Process2",
},
},
},
want: []string{"Process2"},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := &Project{
Version: tt.fields.Version,
LogLocation: tt.fields.LogLocation,
Processes: tt.fields.Processes,
Environment: tt.fields.Environment,
runningProcesses: tt.fields.runningProcesses,
}
got, err := p.GetDependenciesOrderNames()
if (err != nil) != tt.wantErr {
t.Errorf("Project.GetDependenciesOrderNames() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("Project.GetDependenciesOrderNames() = %v, want %v", got, tt.want)
}
})
}
}

25
src/system_test.go Normal file
View File

@ -0,0 +1,25 @@
package main
import (
"path/filepath"
"testing"
)
func getFixtures() []string {
matches, err := filepath.Glob("../fixtures/process-compose-*.yaml")
if err != nil {
panic("no fixtures found")
}
return matches
}
func TestSystem_TestFixtures(t *testing.T) {
fixtures := getFixtures()
for _, fixture := range fixtures {
t.Run(fixture, func(t *testing.T) {
project := createProject(fixture)
project.Run()
})
}
}

14
test_loop.bash Executable file
View File

@ -0,0 +1,14 @@
#!/usr/bin/env bash
LOOPS=3
for (( i=1; i<=LOOPS; i++ ))
do
sleep 1
if [[ -z "${PRINT_ERR}" ]]; then
echo "test loop $i $1 $ABC"
else
echo "test loop $i this is error $1 $PC_PROC_NAME" >&2
fi
done
exit $EXIT_CODE