Use sqlite and static binary (#1837)

This is a rework of #1798 to facilitate a simpler web stack.

# Demo

View http://swarmgame.net/

NOTE: Requires IPv6

# Motivation

Hosting cost is a main motivation.  Cost per month for an EC2 instance, RDS, and the requisite other services approaches >$50 per month.  In contrast, the lowest-tier Lightsail instance is $3.50/month.

The deployment process is of course simplified.

An incidental benefit to using SQLite is reduced latency of web requests; we no longer need to fetch credentials from an AWS API to connect to Postgres.

## Changes

Major changes:
* Use `sqlite` instead of `postgres`
* Use Docker to build a statically-linked deployable binary, rather than deploying the app within a Docker image

Fortunately, the API of `sqlite-simple` is near-identical to that of `postgresql-simple`, so most of the code change there is just to rip out AWS-specific stuff and Postgres connection info.  I have no hesitation to delete this code since if we ever want to use the previous stack again, we can just look at #1798.
This commit is contained in:
Karl Ostmo 2024-05-12 13:45:08 -07:00 committed by GitHub
parent bc0c4040c5
commit c993d9dfdd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
29 changed files with 220 additions and 608 deletions

View File

@ -3,15 +3,12 @@
module Main where
import Control.Monad.Trans.Reader (runReaderT)
import Data.IORef (newIORef)
import Data.Maybe (fromMaybe)
import Network.Wai.Handler.Warp (Port)
import Options.Applicative
import Swarm.Game.State (Sha1 (..))
import Swarm.Web.Tournament
import Swarm.Web.Tournament.Database.Query
import System.Environment (lookupEnv)
import System.Posix.User (getEffectiveUserName)
data AppOpts = AppOpts
{ userWebPort :: Maybe Port
@ -57,25 +54,14 @@ cliInfo =
<> fullDesc
)
deduceConnType :: Bool -> IO DbConnType
deduceConnType isLocalSocketConn =
if isLocalSocketConn
then LocalDBOverSocket . Username <$> getEffectiveUserName
else do
maybeDbPassword <- lookupEnv envarPostgresPasswordKey
case maybeDbPassword of
Just dbPasswordEnvar -> return $ LocalDBFromDockerOverNetwork $ Password dbPasswordEnvar
Nothing -> RemoteDB <$> newIORef Nothing
main :: IO ()
main = do
opts <- execParser cliInfo
connType <- deduceConnType $ isLocalSocketConnection opts
webMain
(AppData (gameGitVersion opts) (persistenceFunctions connType) connType)
(AppData (gameGitVersion opts) persistenceFunctions)
(fromMaybe defaultPort $ userWebPort opts)
where
persistenceFunctions connMode =
persistenceFunctions =
PersistenceLayer
{ lookupScenarioFileContent = withConnInfo lookupScenarioContent
, scenarioStorage =
@ -90,10 +76,5 @@ main = do
}
}
where
withConnInfo f x = do
-- This gets deferred and re-executed upon each invocation
-- of a DB interaction function.
-- We need this behavior because the password fetched via API
-- expires after 15 min.
connInfo <- mkConnectInfo connMode
runReaderT (f x) connInfo
withConnInfo f x =
runReaderT (f x) databaseFilename

View File

@ -1,7 +1,5 @@
#!/bin/bash -ex
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
cd $SCRIPT_DIR/..
cd $(git rev-parse --show-toplevel)
# See https://github.com/swarm-game/swarm/issues/936
STACK_WORK=.stack-work-test stack test --fast "$@"
cabal test --test-show-details=direct -O0 -j "$@"

View File

@ -71,7 +71,6 @@ defaultSolutionTimeout = SolutionTimeout 15
data AppData = AppData
{ swarmGameGitVersion :: Sha1
, persistence :: PersistenceLayer
, dbConnType :: DbConnType
}
type TournamentAPI =
@ -126,10 +125,10 @@ mkApp appData =
:<|> uploadSolution appData
:<|> getScenarioMetadata appData
:<|> downloadRedactedScenario appData
:<|> listScenarios appData
:<|> listScenarios
uploadScenario :: AppData -> MultipartData Mem -> Handler ScenarioCharacterization
uploadScenario (AppData gameVersion persistenceLayer _) multipartData =
uploadScenario (AppData gameVersion persistenceLayer) multipartData =
Handler . withExceptT toServantError . ExceptT $
validateScenarioUpload
args
@ -144,7 +143,7 @@ uploadScenario (AppData gameVersion persistenceLayer _) multipartData =
(scenarioStorage persistenceLayer)
uploadSolution :: AppData -> MultipartData Mem -> Handler SolutionFileCharacterization
uploadSolution (AppData _ persistenceLayer _) multipartData =
uploadSolution (AppData _ persistenceLayer) multipartData =
Handler . withExceptT toServantError . ExceptT $
validateSubmittedSolution
args
@ -159,7 +158,7 @@ uploadSolution (AppData _ persistenceLayer _) multipartData =
(solutionStorage persistenceLayer)
getScenarioMetadata :: AppData -> Sha1 -> Handler ScenarioMetadata
getScenarioMetadata (AppData _ persistenceLayer _) scenarioSha1 =
getScenarioMetadata (AppData _ persistenceLayer) scenarioSha1 =
Handler . withExceptT toServantError $ do
doc <-
ExceptT $
@ -170,7 +169,7 @@ getScenarioMetadata (AppData _ persistenceLayer _) scenarioSha1 =
return $ view scenarioMetadata s
downloadRedactedScenario :: AppData -> Sha1 -> Handler TL.Text
downloadRedactedScenario (AppData _ persistenceLayer _) scenarioSha1 = do
downloadRedactedScenario (AppData _ persistenceLayer) scenarioSha1 = do
Handler . withExceptT toServantError $ do
doc <-
ExceptT $
@ -183,12 +182,12 @@ downloadRedactedScenario (AppData _ persistenceLayer _) scenarioSha1 = do
encodeWith defaultEncodeOptions redactedDict
-- NOTE: This is currently the only API endpoint that invokes
-- 'mkConnectInfo' directly
listScenarios :: AppData -> Handler [TournamentGame]
listScenarios (AppData _ _ connMode) =
Handler $ liftIO $ do
connInfo <- mkConnectInfo connMode
runReaderT listGames connInfo
-- 'runReaderT' directly
listScenarios :: Handler [TournamentGame]
listScenarios =
Handler $
liftIO $
runReaderT listGames databaseFilename
-- * Web app declaration

View File

@ -10,28 +10,24 @@
-- SQL Queries for Swarm tournaments.
module Swarm.Web.Tournament.Database.Query where
import Control.Monad (guard)
import Control.Monad.IO.Class (liftIO)
import Control.Monad.Trans.Maybe (MaybeT (..), runMaybeT)
import Control.Monad.Trans.Reader (ReaderT, ask)
import Data.ByteString.Lazy qualified as LBS
import Data.IORef
import Data.Maybe (listToMaybe)
import Data.String.Utils (strip)
import Data.Time.Clock
import Database.PostgreSQL.Simple
import Database.PostgreSQL.Simple.FromRow
import Database.PostgreSQL.Simple.ToField
import Database.SQLite.Simple
import Database.SQLite.Simple.ToField
import Swarm.Game.Scenario.Scoring.CodeSize
import Swarm.Game.State (Sha1 (..))
import Swarm.Game.Tick (TickNumber (..))
import Swarm.Web.Tournament.Type
import System.Exit (ExitCode (..))
import System.Process
-- | Used for local development only
envarPostgresPasswordKey :: String
envarPostgresPasswordKey = "LOCAL_PGPASS"
type ConnectInfo = String
databaseFilename :: ConnectInfo
databaseFilename = "swarm-games.db"
newtype UserId = UserId Int
@ -117,90 +113,6 @@ data DbConnType
tokenRefreshInterval :: NominalDiffTime
tokenRefreshInterval = 10 * 60
genNewToken :: ConnectInfo -> IO (Either String String)
genNewToken ci = do
(exitCode, stdoutString, stderrString) <-
readProcessWithExitCode
"aws"
[ "rds"
, "generate-db-auth-token"
, "--hostname"
, connectHost ci
, "--port"
, show $ connectPort ci
, "--region"
, region
, "--username"
, connectUser ci
]
""
return $ case exitCode of
ExitSuccess -> Right $ strip stdoutString
ExitFailure _ -> Left stderrString
where
region = "us-east-1"
getAwsCredentials :: TokenRef -> ConnectInfo -> IO ConnectInfo
getAwsCredentials tokRef ci = do
currTime <- getCurrentTime
maybePreviousTok <- readIORef tokRef
let maybeStillValidTok = case maybePreviousTok of
Nothing -> Nothing
Just (TokenWithExpiration exprTime tok) ->
guard (currTime < exprTime) >> Just tok
case maybeStillValidTok of
Just (Password tok) ->
return $
ci
{ connectPassword = tok
}
Nothing -> do
eitherNewTok <- genNewToken ci
case eitherNewTok of
Right newTok -> do
let nextExpirationTime = addUTCTime tokenRefreshInterval currTime
atomicWriteIORef tokRef
. Just
. TokenWithExpiration nextExpirationTime
$ Password newTok
return $
ci
{ connectPassword = newTok
}
-- NOTE: This is not exactly valid behavior:
Left _errMsg -> return ci
mkConnectInfo :: DbConnType -> IO ConnectInfo
mkConnectInfo connType = do
let swarmDbConnect =
defaultConnectInfo
{ connectDatabase = "swarm"
}
case connType of
LocalDBFromDockerOverNetwork (Password dbPasswd) ->
return $
swarmDbConnect
{ connectHost = "host.docker.internal"
, connectUser = "swarm-app"
, connectPassword = dbPasswd
}
LocalDBOverSocket (Username username) ->
return
swarmDbConnect
{ connectHost = "/var/run/postgresql"
, connectUser = username
}
RemoteDB tokRef -> getAwsCredentials tokRef rdsConnectionInfo
where
rdsConnectionInfo =
defaultConnectInfo
{ connectHost = "swarm-tournaments.cv6iymakujnb.us-east-1.rds.amazonaws.com"
, connectUser = "swarm-app"
, connectDatabase = "swarm"
}
-- * Authentication
getUserId :: Connection -> UserAlias -> IO UserId
@ -226,13 +138,13 @@ getUserId conn userAlias = do
lookupScenarioContent :: Sha1 -> ReaderT ConnectInfo IO (Maybe LBS.ByteString)
lookupScenarioContent sha1 = do
connInfo <- ask
liftIO . fmap (fmap fromOnly . listToMaybe) . withConnect connInfo $ \conn ->
liftIO . fmap (fmap fromOnly . listToMaybe) . withConnection connInfo $ \conn ->
query conn "SELECT content FROM scenarios WHERE content_sha1 = ?;" (Only sha1)
lookupSolutionSubmission :: Sha1 -> ReaderT ConnectInfo IO (Maybe AssociatedSolutionSolutionCharacterization)
lookupSolutionSubmission contentSha1 = do
connInfo <- ask
liftIO $ withConnect connInfo $ \conn -> runMaybeT $ do
liftIO $ withConnection connInfo $ \conn -> runMaybeT $ do
evaluationId :: Int <-
MaybeT $
fmap fromOnly . listToMaybe
@ -246,14 +158,14 @@ lookupSolutionSubmission contentSha1 = do
lookupScenarioSolution :: Sha1 -> ReaderT ConnectInfo IO (Maybe AssociatedSolutionSolutionCharacterization)
lookupScenarioSolution scenarioSha1 = do
connInfo <- ask
solnChar <- liftIO . fmap listToMaybe . withConnect connInfo $ \conn ->
solnChar <- liftIO . fmap listToMaybe . withConnection connInfo $ \conn ->
query conn "SELECT wall_time_seconds, ticks, seed, char_count, ast_size FROM evaluated_solution WHERE builtin AND scenario = ? LIMIT 1;" (Only scenarioSha1)
return $ AssociatedSolutionSolutionCharacterization scenarioSha1 <$> solnChar
listGames :: ReaderT ConnectInfo IO [TournamentGame]
listGames = do
connInfo <- ask
liftIO $ withConnect connInfo $ \conn ->
liftIO $ withConnection connInfo $ \conn ->
query_ conn "SELECT original_filename, scenario_uploader, scenario, submission_count, swarm_git_sha1 FROM submissions;"
-- * Insertion
@ -263,7 +175,7 @@ insertScenario ::
ReaderT ConnectInfo IO Sha1
insertScenario s = do
connInfo <- ask
h <- liftIO $ withConnect connInfo $ \conn -> do
h <- liftIO $ withConnection connInfo $ \conn -> do
uid <- getUserId conn $ uploader $ upload s
[Only resultList] <-
query
@ -287,7 +199,7 @@ insertSolutionSubmission ::
ReaderT ConnectInfo IO Sha1
insertSolutionSubmission (CharacterizationResponse solutionUpload s (SolutionUploadResponsePayload scenarioSha)) = do
connInfo <- ask
liftIO $ withConnect connInfo $ \conn -> do
liftIO $ withConnection connInfo $ \conn -> do
uid <- getUserId conn $ uploader solutionUpload
solutionEvalId <- insertSolution conn False scenarioSha $ characterization s

View File

@ -10,7 +10,7 @@ module Swarm.Web.Tournament.Type where
import Data.Aeson
import Data.ByteString.Lazy qualified as LBS
import Data.Text qualified as T
import Database.PostgreSQL.Simple.ToField
import Database.SQLite.Simple.ToField
import GHC.Generics (Generic)
import Servant
import Servant.Docs (ToCapture)

View File

@ -462,7 +462,6 @@ library swarm-tournament
other-modules: Paths_swarm
autogen-modules: Paths_swarm
build-depends:
MissingH,
SHA,
aeson,
base,
@ -474,11 +473,10 @@ library swarm-tournament
http-types,
lens,
mtl,
postgresql-simple >=0.7 && <0.7.1,
process,
servant-docs,
servant-multipart,
servant-server >=0.19 && <0.21,
sqlite-simple >=0.4.19.0 && <0.4.20,
text,
time,
transformers,
@ -757,7 +755,6 @@ executable swarm-host-tournament
base,
optparse-applicative >=0.16 && <0.19,
transformers,
unix,
warp,
build-depends:

View File

@ -50,8 +50,6 @@ main = do
Tournament.AppData
{ Tournament.swarmGameGitVersion = Sha1 "abcdef"
, Tournament.persistence = mkPersistenceLayer scenariosMap
, -- NOTE: This is not actually used/exercised by the tests:
Tournament.dbConnType = LocalDBOverSocket $ Username ""
}
type LocalFileLookup = NEMap Sha1 FilePathAndContent

23
tournament/README.md Normal file
View File

@ -0,0 +1,23 @@
# Usage
## Installation prerequisites:
Install sqlite:
```
sudo apt install sqlite3
```
## Deployment
Run this script (requires Docker):
```
tournament/scripts/docker/build-static-binary.sh
```
# Testing
## Unit tests
```
scripts/test/run-tests.sh swarm:test:tournament-host
```

View File

@ -1,267 +0,0 @@
--
-- PostgreSQL database dump
--
-- Dumped from database version 14.11 (Ubuntu 14.11-0ubuntu0.22.04.1)
-- Dumped by pg_dump version 14.11 (Ubuntu 14.11-0ubuntu0.22.04.1)
SET statement_timeout = 0;
SET lock_timeout = 0;
SET idle_in_transaction_session_timeout = 0;
SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
SELECT pg_catalog.set_config('search_path', '', false);
SET check_function_bodies = false;
SET xmloption = content;
SET client_min_messages = warning;
SET row_security = off;
--
-- Name: swarm; Type: DATABASE; Schema: -; Owner: postgres
--
CREATE DATABASE swarm WITH TEMPLATE = template0 ENCODING = 'UTF8' LOCALE = 'en_US.UTF-8';
ALTER DATABASE swarm OWNER TO postgres;
\connect swarm
SET statement_timeout = 0;
SET lock_timeout = 0;
SET idle_in_transaction_session_timeout = 0;
SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
SELECT pg_catalog.set_config('search_path', '', false);
SET check_function_bodies = false;
SET xmloption = content;
SET client_min_messages = warning;
SET row_security = off;
SET default_tablespace = '';
SET default_table_access_method = heap;
--
-- Name: evaluated_solution; Type: TABLE; Schema: public; Owner: kostmo
--
CREATE TABLE public.evaluated_solution (
id integer NOT NULL,
evaluated_at timestamp with time zone DEFAULT now() NOT NULL,
scenario character varying(40) NOT NULL,
seed bigint NOT NULL,
wall_time_seconds double precision NOT NULL,
ticks bigint,
char_count integer,
ast_size integer,
builtin boolean NOT NULL
);
ALTER TABLE public.evaluated_solution OWNER TO kostmo;
--
-- Name: scenarios; Type: TABLE; Schema: public; Owner: kostmo
--
CREATE TABLE public.scenarios (
content_sha1 character varying(40) NOT NULL,
uploader integer NOT NULL,
original_filename text,
swarm_git_sha1 character varying(40),
uploaded_at timestamp with time zone DEFAULT now() NOT NULL,
content text NOT NULL
);
ALTER TABLE public.scenarios OWNER TO kostmo;
--
-- Name: solution_id_seq; Type: SEQUENCE; Schema: public; Owner: kostmo
--
CREATE SEQUENCE public.solution_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER TABLE public.solution_id_seq OWNER TO kostmo;
--
-- Name: solution_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: kostmo
--
ALTER SEQUENCE public.solution_id_seq OWNED BY public.evaluated_solution.id;
--
-- Name: solution_submission; Type: TABLE; Schema: public; Owner: kostmo
--
CREATE TABLE public.solution_submission (
content_sha1 character varying(40) NOT NULL,
uploader integer NOT NULL,
uploaded_at timestamp with time zone DEFAULT now() NOT NULL,
solution_evaluation integer
);
ALTER TABLE public.solution_submission OWNER TO kostmo;
--
-- Name: users; Type: TABLE; Schema: public; Owner: kostmo
--
CREATE TABLE public.users (
id integer NOT NULL,
alias text NOT NULL,
created_at timestamp with time zone DEFAULT now() NOT NULL
);
ALTER TABLE public.users OWNER TO kostmo;
--
-- Name: submissions; Type: VIEW; Schema: public; Owner: kostmo
--
CREATE VIEW public.submissions AS
SELECT scenarios.original_filename,
scenarios.content_sha1 AS scenario,
scenarios.uploaded_at AS scenario_uploaded_at,
COALESCE(foo.submission_count, (0)::bigint) AS submission_count,
users.alias AS scenario_uploader,
scenarios.swarm_git_sha1
FROM ((public.scenarios
LEFT JOIN ( SELECT evaluated_solution.scenario,
count(*) AS submission_count
FROM public.evaluated_solution
WHERE (NOT evaluated_solution.builtin)
GROUP BY evaluated_solution.scenario) foo ON (((scenarios.content_sha1)::text = (foo.scenario)::text)))
JOIN public.users ON ((scenarios.uploader = users.id)));
ALTER TABLE public.submissions OWNER TO kostmo;
--
-- Name: users_id_seq; Type: SEQUENCE; Schema: public; Owner: kostmo
--
CREATE SEQUENCE public.users_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER TABLE public.users_id_seq OWNER TO kostmo;
--
-- Name: users_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: kostmo
--
ALTER SEQUENCE public.users_id_seq OWNED BY public.users.id;
--
-- Name: evaluated_solution id; Type: DEFAULT; Schema: public; Owner: kostmo
--
ALTER TABLE ONLY public.evaluated_solution ALTER COLUMN id SET DEFAULT nextval('public.solution_id_seq'::regclass);
--
-- Name: users id; Type: DEFAULT; Schema: public; Owner: kostmo
--
ALTER TABLE ONLY public.users ALTER COLUMN id SET DEFAULT nextval('public.users_id_seq'::regclass);
--
-- Name: scenarios scenarios_pkey; Type: CONSTRAINT; Schema: public; Owner: kostmo
--
ALTER TABLE ONLY public.scenarios
ADD CONSTRAINT scenarios_pkey PRIMARY KEY (content_sha1);
--
-- Name: solution_submission solution_file_pkey; Type: CONSTRAINT; Schema: public; Owner: kostmo
--
ALTER TABLE ONLY public.solution_submission
ADD CONSTRAINT solution_file_pkey PRIMARY KEY (content_sha1);
--
-- Name: evaluated_solution solution_pkey; Type: CONSTRAINT; Schema: public; Owner: kostmo
--
ALTER TABLE ONLY public.evaluated_solution
ADD CONSTRAINT solution_pkey PRIMARY KEY (id);
--
-- Name: users users_pkey; Type: CONSTRAINT; Schema: public; Owner: kostmo
--
ALTER TABLE ONLY public.users
ADD CONSTRAINT users_pkey PRIMARY KEY (id);
--
-- Name: fki_solution_file_solution; Type: INDEX; Schema: public; Owner: kostmo
--
CREATE INDEX fki_solution_file_solution ON public.solution_submission USING btree (solution_evaluation);
--
-- Name: fki_solution_scenario; Type: INDEX; Schema: public; Owner: kostmo
--
CREATE INDEX fki_solution_scenario ON public.evaluated_solution USING btree (scenario);
--
-- Name: scenario_uploader; Type: INDEX; Schema: public; Owner: kostmo
--
CREATE INDEX scenario_uploader ON public.scenarios USING btree (uploader);
--
-- Name: scenarios scenarios_uploader_fkey; Type: FK CONSTRAINT; Schema: public; Owner: kostmo
--
ALTER TABLE ONLY public.scenarios
ADD CONSTRAINT scenarios_uploader_fkey FOREIGN KEY (uploader) REFERENCES public.users(id) NOT VALID;
--
-- Name: solution_submission solution_file_solution; Type: FK CONSTRAINT; Schema: public; Owner: kostmo
--
ALTER TABLE ONLY public.solution_submission
ADD CONSTRAINT solution_file_solution FOREIGN KEY (solution_evaluation) REFERENCES public.evaluated_solution(id) NOT VALID;
--
-- Name: evaluated_solution solution_scenario; Type: FK CONSTRAINT; Schema: public; Owner: kostmo
--
ALTER TABLE ONLY public.evaluated_solution
ADD CONSTRAINT solution_scenario FOREIGN KEY (scenario) REFERENCES public.scenarios(content_sha1) NOT VALID;
--
-- PostgreSQL database dump complete
--

View File

@ -0,0 +1,54 @@
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "users" (
"id" INTEGER NOT NULL UNIQUE,
"alias" TEXT NOT NULL,
"created_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY("id" AUTOINCREMENT)
);
CREATE TABLE IF NOT EXISTS "scenarios" (
"content_sha1" TEXT NOT NULL UNIQUE,
"uploader" INTEGER NOT NULL,
"original_filename" TEXT,
"swarm_git_sha1" TEXT,
"uploaded_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
"content" TEXT NOT NULL,
PRIMARY KEY("content_sha1"),
FOREIGN KEY(uploader) REFERENCES users(id)
);
CREATE TABLE IF NOT EXISTS "evaluated_solution" (
"id" INTEGER NOT NULL UNIQUE,
"evaluated_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
"scenario" TEXT NOT NULL,
"seed" INTEGER NOT NULL,
"wall_time_seconds" REAL NOT NULL,
"ticks" INTEGER,
"char_count" INTEGER,
"ast_size" INTEGER,
"builtin" BOOLEAN,
PRIMARY KEY("id" AUTOINCREMENT),
FOREIGN KEY(scenario) REFERENCES scenarios(content_sha1)
);
CREATE TABLE IF NOT EXISTS "solution_submission" (
"content_sha1" TEXT NOT NULL,
"uploader" INTEGER NOT NULL,
"uploaded_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
"solution_evaluation" INTEGER,
PRIMARY KEY("content_sha1"),
FOREIGN KEY(solution_evaluation) REFERENCES evaluated_solution(id)
);
CREATE VIEW submissions AS
SELECT scenarios.original_filename,
scenarios.content_sha1 AS scenario,
scenarios.uploaded_at AS scenario_uploaded_at,
COALESCE(foo.submission_count, 0) AS submission_count,
users.alias AS scenario_uploader,
scenarios.swarm_git_sha1
FROM ((scenarios
LEFT JOIN ( SELECT evaluated_solution.scenario,
count(*) AS submission_count
FROM evaluated_solution
WHERE (NOT evaluated_solution.builtin)
GROUP BY evaluated_solution.scenario) foo ON (scenarios.content_sha1 = foo.scenario))
JOIN users ON (scenarios.uploader = users.id));
COMMIT;

View File

@ -2,4 +2,4 @@
GIT_ROOT_DIR=$(git rev-parse --show-toplevel)
pg_dump --create -s -d swarm > $GIT_ROOT_DIR/tournament/schema/schema-local.sql
sqlite3 swarm-games.db '.schema' > $GIT_ROOT_DIR/tournament/schema/swarm-sqlite-schema.sql

View File

@ -2,7 +2,4 @@
GIT_ROOT_DIR=$(git rev-parse --show-toplevel)
sudo service postgresql restart
dropdb swarm
sudo -u postgres psql < $GIT_ROOT_DIR/tournament/schema/schema-local.sql
sqlite3 swarm-games.db < $GIT_ROOT_DIR/tournament/schema/swarm-sqlite-schema.sql

View File

@ -1,27 +1,11 @@
# Running in local development environment
The `client.sh` script can be run with either the `server-docker.sh` or the `server-native.sh` script as the host.
The `client.sh` script can be run with the `server-native.sh` script as the host.
Running the server application natively is the simplest option and connects to the local Postgres database via a socket.
Running the server inside a local Docker image requires supplying the Postgres password as an environment variable.
Running the server application natively is the simplest option and connects to the local database file.
## Database setup
One first needs to install a local Postgres server.
After configuring logins and users, one may populate the database using the stored `schema-local.sql` schema with a script:
One may populate the database using the committed schema with a script:
tournament/scripts/database/recreate-local-database.sh
### Configuring database access from Docker
See this answer: https://stackoverflow.com/a/58015643/105137
To summarize:
* Edit `postgresql.conf`, uncomment and set `listen_addresses = '*'`
* Edit `pg_hba.conf`, add the line:
```
host all all 172.17.0.0/16 password
```

View File

@ -2,7 +2,9 @@
cd $(git rev-parse --show-toplevel)
HOST=${1:-localhost:8080}
tournament/scripts/demo/client/submit.sh \
localhost:8008 \
$HOST \
data/scenarios/Challenges/arbitrage.yaml \
data/scenarios/Challenges/_arbitrage/solution.sw

View File

@ -5,9 +5,11 @@
cd $(git rev-parse --show-toplevel)
tournament/scripts/demo/client/test-cases/local/good-submit.sh
HOST=${1:-localhost:8080}
tournament/scripts/demo/client/test-cases/local/good-submit.sh $HOST
tournament/scripts/demo/client/submit.sh \
localhost:8008 \
$HOST \
data/scenarios/Challenges/dimsum.yaml \
data/scenarios/Challenges/_arbitrage/solution.sw

View File

@ -1,14 +0,0 @@
#!/bin/bash -ex
GIT_ROOT_DIR=$(git rev-parse --show-toplevel)
cd $GIT_ROOT_DIR
# NOTE: First, you may need to build the Docker image
tournament/scripts/docker/build-image.sh
docker run \
--add-host=host.docker.internal:host-gateway \
--env-file $GIT_ROOT_DIR/tournament/scripts/docker/local-pg-credentials.env \
-it \
-p 8080:8080 \
--rm swarm

View File

@ -6,8 +6,7 @@ cd $(git rev-parse --show-toplevel)
GIT_HASH=$(git rev-parse HEAD)
stack build --fast swarm:swarm-host-tournament && \
stack exec swarm-host-tournament -- \
cabal run -j -O0 swarm:swarm-host-tournament -- \
--native-dev \
--port 8080 \
--version $GIT_HASH \

View File

@ -0,0 +1,17 @@
#!/bin/bash -ex
cd $(git rev-parse --show-toplevel)
tournament/scripts/docker/build-static-binary.sh
INTERNAL_BINARY_NAME=tournament-bin
scp -r $INTERNAL_BINARY_NAME lightsail:
rm $INTERNAL_BINARY_NAME
CURRENT_GIT_HASH_FILEPATH=git-hash.txt
git rev-parse HEAD > $CURRENT_GIT_HASH_FILEPATH
scp $CURRENT_GIT_HASH_FILEPATH lightsail:
rm $CURRENT_GIT_HASH_FILEPATH
ssh lightsail -C 'sudo systemctl restart swarm-tournament'

View File

@ -0,0 +1,6 @@
#!/bin/bash -ex
cd $(git rev-parse --show-toplevel)
scp -r tournament/web lightsail:tournament
scp -r data lightsail:.local/share/swarm

View File

@ -1,101 +0,0 @@
# This is meant to be invoked while
# the CWD is the swarm repository root.
FROM amazonlinux:latest as amz
LABEL org.opencontainers.image.authors="Karl Ostmo <kostmo@gmail.com>"
ENV TZ=America/Los_Angeles
# OS dependencies
RUN yum -y update && yum -y install \
postgresql-devel
# The 'python' executable is required to run the AWS CLI installer
#RUN yum -y install python
RUN ln -s /usr/bin/python3 /usr/bin/python
RUN curl https://s3.amazonaws.com/aws-cli/awscli-bundle.zip -o awscli-bundle.zip
RUN unzip awscli-bundle.zip
RUN ./awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws
RUN mkdir -p /opt/swarm
FROM amz as system-build-deps
# These packages are only needed at build time, not at runtime.
RUN yum -y install \
ncurses-compat-libs \
gmp \
gmp-devel \
zlib-devel \
fftw3-devel \
xz-devel \
nss \
nss-devel \
openssl-devel \
gcc \
gcc-c++ \
make \
tar
FROM system-build-deps as haskell-compilation-layer
# install ghcup
RUN \
curl https://downloads.haskell.org/~ghcup/x86_64-linux-ghcup > /usr/bin/ghcup && \
chmod +x /usr/bin/ghcup
ARG GHC=9.6.4
# install GHC and cabal
RUN \
ghcup -v install ghc --isolate /usr/local --force ${GHC} && \
ghcup -v install cabal --isolate /usr/local/bin
WORKDIR /opt/swarm
COPY ./swarm.cabal /opt/swarm/swarm.cabal
RUN cabal update
# Must manually list the transitive closure of "internal" dependencies (sublibraries)
# of our executable.
# Note that we avoid simply listing all sublibraries here
# (i.e. scripts/gen/list-sublibraries.sh) because that
# includes 'swarm:swarm-web' and will build pandoc.
RUN cabal build --only-dependencies \
swarm:swarm-host-tournament \
swarm:swarm-tournament \
swarm:swarm-engine \
swarm:swarm-lang \
swarm:swarm-scenario \
swarm:swarm-util
COPY ./src /opt/swarm/src
COPY ./app /opt/swarm/app
# The following are not strictly needed for compiling the
# selected dependencies, but 'cabal build' spews warnings
# when they are absent
COPY ./test /opt/swarm/test
COPY ./CHANGELOG.md /opt/swarm/CHANGELOG.md
COPY ./LICENSE /opt/swarm/LICENSE
COPY tournament/scripts/docker/build-server-executable.sh /opt/swarm/build-server-executable.sh
RUN /opt/swarm/build-server-executable.sh /opt/swarm/tournament-bin
FROM amz
COPY --from=haskell-compilation-layer /opt/swarm/tournament-bin /opt/swarm/tournament-bin
COPY ./data /root/.local/share/swarm/data
COPY ./tournament/web /root/tournament/web
# This was produced by the parent script, 'build-image.sh'.
COPY ./git-hash.txt /root/git-hash.txt
EXPOSE 8080
# We begin initially with CWD as the filesystem root, "/".
# We first 'cd' into the home directory, which is "/root", so we
# have access to the static web files.
CMD cd && /opt/swarm/tournament-bin --port 8080 --version $(cat git-hash.txt)

View File

@ -0,0 +1,57 @@
# This is meant to be invoked while
# the CWD is the swarm repository root.
FROM quay.io/benz0li/ghc-musl:9.6.4 as hs
LABEL org.opencontainers.image.authors="Karl Ostmo <kostmo@gmail.com>"
ENV TZ=America/Los_Angeles
RUN \
apk add --no-cache git curl gcc g++ gmp-dev ncurses-dev libffi-dev make xz tar perl && \
apk add --no-cache zlib zlib-dev zlib-static ncurses-static
# install ghcup
RUN \
curl https://downloads.haskell.org/~ghcup/x86_64-linux-ghcup > /usr/bin/ghcup && \
chmod +x /usr/bin/ghcup
ARG GHC=9.6.4
# install GHC and cabal
RUN ghcup -v install ghc --isolate /usr/local --force ${GHC}
RUN mkdir -p /opt/swarm
FROM hs as system-build-deps
WORKDIR /opt/swarm
COPY ./swarm.cabal /opt/swarm/swarm.cabal
RUN cabal update
# Must manually list the transitive closure of "internal" dependencies (sublibraries)
# of our executable.
# Note that we avoid simply listing all sublibraries here
# (i.e. scripts/gen/list-sublibraries.sh) because that
# includes 'swarm:swarm-web' and will build pandoc.
RUN cabal build --only-dependencies \
swarm:swarm-host-tournament \
swarm:swarm-tournament \
swarm:swarm-engine \
swarm:swarm-lang \
swarm:swarm-scenario \
swarm:swarm-util
COPY ./src /opt/swarm/src
COPY ./app /opt/swarm/app
# The following are not strictly needed for compiling the
# selected dependencies, but 'cabal build' spews warnings
# when they are absent
COPY ./test /opt/swarm/test
COPY ./CHANGELOG.md /opt/swarm/CHANGELOG.md
COPY ./LICENSE /opt/swarm/LICENSE
COPY ./tournament/scripts/docker/build-server-executable.sh /opt/swarm/build-server-executable.sh
RUN cd /opt/swarm && ./build-server-executable.sh /opt/swarm/tournament-bin

View File

@ -1,5 +0,0 @@
#!/bin/bash -ex
cd $(git rev-parse --show-toplevel)
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 254464607561.dkr.ecr.us-east-1.amazonaws.com/swarm-game

View File

@ -1,11 +0,0 @@
#!/bin/bash -ex
cd $(git rev-parse --show-toplevel)
CURRENT_GIT_HASH_FILEPATH=git-hash.txt
git rev-parse HEAD > $CURRENT_GIT_HASH_FILEPATH
docker build --tag swarm --file tournament/scripts/docker/Dockerfile .
rm $CURRENT_GIT_HASH_FILEPATH

View File

@ -1,4 +1,4 @@
#!/bin/bash -ex
#!/bin/sh -ex
# Usage:
# Intended to be invoked in Dockerfile.
@ -8,9 +8,8 @@
# Note that we use 'cabal' instead of 'stack' becuase
# 'stack' fails to compile the 'vty' package within the Amazon Linux docker image.
# For faster development iteration, disable optimizations:
CABAL_ARGS="--disable-optimization swarm:swarm-host-tournament"
#CABAL_ARGS="swarm:swarm-host-tournament"
BUILD_TARGET=swarm:swarm-host-tournament
CABAL_ARGS="-j -O0 --enable-executable-static $BUILD_TARGET"
cabal build -j $CABAL_ARGS
cp $(cabal list-bin $CABAL_ARGS) $1
cabal build $CABAL_ARGS
cp $(cabal list-bin $CABAL_ARGS) $1

View File

@ -0,0 +1,14 @@
#!/bin/bash -ex
cd $(git rev-parse --show-toplevel)
TAG_NAME=swarm
docker build --tag $TAG_NAME --file tournament/scripts/docker/alpine/Dockerfile .
INTERNAL_BINARY_NAME=tournament-bin
ID=$(docker create $TAG_NAME)
docker cp $ID:/opt/swarm/$INTERNAL_BINARY_NAME - | tar xv
docker rm -v $ID
strip $INTERNAL_BINARY_NAME

View File

@ -1,6 +0,0 @@
#!/bin/bash
# Run this script on your host machine to prepare
# for development with Docker
sudo apt install docker.io

View File

@ -1,6 +0,0 @@
#!/bin/bash -ex
cd $(git rev-parse --show-toplevel)
#$NAME_ARG="--name swarm-container"
docker run -it $NAME_ARG -p 8080:8080 --entrypoint /bin/bash swarm

View File

@ -1 +0,0 @@
LOCAL_PGPASS=irrelevantpassword

View File

@ -1,16 +0,0 @@
#!/bin/bash -ex
cd $(git rev-parse --show-toplevel)
tournament/scripts/docker/build-image.sh
AWS_DOCKER_IMAGE=254464607561.dkr.ecr.us-east-1.amazonaws.com/swarm-game:latest
docker tag swarm:latest $AWS_DOCKER_IMAGE
# Optionally log in again
tournament/scripts/docker/aws-login.sh
docker push $AWS_DOCKER_IMAGE
# Next, run:
# eb deploy swarm-tournament-server-env