add script and now.sh function to edit pg_dump output (close #1861) (#1884)

<!-- Thank you for submitting this PR! :) -->
<!-- Provide a general summary of your changes in the Title above ^, end with (close #<issue-no>) or (fix #<issue-no>) -->

### Description
<!-- The title might not be enough to convey how this change affects the user. -->
<!-- Describe the changes from a user's perspective -->
This PR adds a bash script and a serverless function to clean up the output of `pg_dump` so that it can be used as a migration file for Hasura. This can be later integrated with the CLI so that the cleanup is handled by CLI.

### Affected components 
<!-- Remove non-affected components from the list -->

- Scripts

### Related Issues
<!-- Please make sure you have an issue associated with this Pull Request -->
<!-- And then add `(close #<issue-no>)` to the pull request title -->
<!-- Add the issue number below (e.g. #234) -->
#1861 
### Solution and Design
<!-- How is this issue solved/fixed? What is the design? -->
<!-- It's better if we elaborate -->
- A serverless function written in Go gets the SQL content though HTTP POST.
- A set of pre-defined lines are removed from this SQL string.
- SQL comments are removed using regex matching.
- Postgres triggers created by Hasura for use with event triggers are removed with regex matching.
- Empty newlines are removed by regex matching.
- Resulting string is returned in the HTTP response.
### Steps to test and verify
<!-- If this is a feature, what are the steps to try them out? -->
<!-- If this is a bug-fix, how do we verify the fix? -->
```bash
curl --data-binary @filename.sql https://hasura-edit-pg-dump.now.sh > newfile.sql
```

### Limitations, known bugs & workarounds
<!-- Limitations of the PR, known bugs and suggested workarounds -->
NA
<!-- Feel free to delete these comment lines -->
This commit is contained in:
Shahidh K Muhammed 2019-03-27 21:36:58 +05:30 committed by GitHub
parent c7fdf013fa
commit 42a25f044e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 167 additions and 0 deletions

View File

@ -0,0 +1,23 @@
# hasura-edit-pg-dump
This is a serverless function written in Go and deployed onto now.sh which can
takes the output SQL from `pg_dump` and clean it up so that it can be used as a
migration file with Hasura.
1. SQL front matter, like `SET` statements are removed.
2. Comments and empty newlines are removed.
3. Postgres triggers created by Hasura are removed.
This app is available at https://hasura-edit-pg-dump.now.sh
Usage:
```bash
curl --data-binary @file.sql https://hasura-edit-pg-dump.now.sh > cleaned.sql
```
A bash version is also available:
```bash
./edit-pg-dump.sh filename.sql
```

View File

@ -0,0 +1,54 @@
#! /usr/bin/env bash
#
# Usage: ./edit-pg-dump.sh <file-name.sql>
#
if [ "$#" -ne 1 ]; then
echo "Invalid usage: ./edit-pg-dump.sh <file-name.sql>"
fi
filename=$1
if [ ! -f $filename ] || [ "$filename" == "" ]; then
echo "file $filename does not exist"
exit 1
fi
echo "making a copy"
cp $filename $filename.backup
echo "processing file"
# delete all comments
sed -i '/^--/d' $filename
# delete front matter
read -r -d '' lines << EOF
SET statement_timeout = 0;
SET lock_timeout = 0;
SET idle_in_transaction_session_timeout = 0;
SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
SELECT pg_catalog.set_config('search_path', '', false);
SET check_function_bodies = false;
SET client_min_messages = warning;
SET row_security = off;
SET default_tablespace = '';
SET default_with_oids = false;
EOF
while read -r line; do
sed -i '/^'"$line"'$/d' $filename
done <<< $lines
# delete notify triggers
sed -i -E '/^CREATE TRIGGER "?notify_hasura_\w+"? AFTER \w+ ON "?\w+"?\."?\w+"? FOR EACH ROW EXECUTE PROCEDURE "?hdb_views"?\."?notify_hasura_\w+"?\(\);$/d' $filename
# delete empty lines
sed -i '/^[[:space:]]*$/d' $filename
echo "done"

View File

@ -0,0 +1,84 @@
package main
import (
"fmt"
"io/ioutil"
"net/http"
"regexp"
"strings"
)
var frontMatter = []string{
"SET statement_timeout = 0;",
"SET lock_timeout = 0;",
"SET idle_in_transaction_session_timeout = 0;",
"SET client_encoding = 'UTF8';",
"SET standard_conforming_strings = on;",
"SELECT pg_catalog.set_config('search_path', '', false);",
"SET check_function_bodies = false;",
"SET client_min_messages = warning;",
"SET row_security = off;",
"SET default_tablespace = '';",
"SET default_with_oids = false;",
}
const helpStr = `POST the SQL file contents to this URL:
curl --data-binary @filename.sql https://hasura-edit-pg-dump.now.sh > newfile.sql`
// Handler is the now.sh serverless handler
func Handler(w http.ResponseWriter, r *http.Request) {
// if method is not POST, respond with help message
if r.Method != "POST" {
fmt.Fprintf(w, helpStr)
return
}
// build the regular expression for matching empty newlines
emptyNewlineRe, err := regexp.Compile(`(?m)^\s*$[\r\n]*`)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
// build the regular expression for matching SQL comments
commentsRe, err := regexp.Compile(`(?m)^--.*$`)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
// build the regular expression for matching triggers created by Hasura
triggerRe, err := regexp.Compile(`(?m)^CREATE TRIGGER "?notify_hasura_\w+"? AFTER \w+ ON "?\w+"?\."?\w+"? FOR EACH ROW EXECUTE PROCEDURE "?hdb_views"?\."?notify_hasura_\w+"?\(\);$`)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
// read the POST body
body, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
// convert bytes to string
bodyStr := string(body)
// remove the SQL front matter
for _, l := range frontMatter {
bodyStr = strings.Replace(bodyStr, l, "", 1)
}
// replace the regex matches
bodyStr = commentsRe.ReplaceAllLiteralString(bodyStr, "")
bodyStr = triggerRe.ReplaceAllLiteralString(bodyStr, "")
bodyStr = emptyNewlineRe.ReplaceAllLiteralString(bodyStr, "\n")
// prefix a message
bodyStr = "--\r\n-- SQL edited by https://hasura-edit-pg-dump.now.sh\r\n--\r\n" + bodyStr
// respond with the edited string
fmt.Fprintf(w, bodyStr)
}

View File

@ -0,0 +1,6 @@
{
"version": 2,
"name": "hasura-edit-pg-dump",
"alias": "hasura-edit-pg-dump.now.sh",
"builds": [{ "src": "index.go", "use": "@now/go" }]
}