5c03fe3 · typeable/oauth2-proxy

git.iko.soy/typeable/oauth2-proxy

Merge pull request #90 from 18F/robots-txt

Provide a robots.txt that denies all crawlers

commit5c03fe384043b0dac5c455e8df4e3b6acf67af7c authorJehiah Czebotar <jehiah@gmail.com> dateSun 10 May 2015, 16:07 (11 years ago) parent26170c5 5b07d9f

3 files changed+28

compare with master

unifiedsplit

README.md+1 oauthproxy.go+11 oauthproxy_test.go+16

README.md+1

@@ -149,6 +149,7 @@ The command line to run `google_auth_proxy` would look like this:

149149

150150Google Auth Proxy responds directly to the following endpoints. All other endpoints will be proxied upstream when authenticated.

151151

152+* /robots.txt - returns a 200 OK response that disallows all User-agents from all paths; see [robotstxt.org](http://www.robotstxt.org/) for more info

152153* /ping - returns an 200 OK response

153154* /oauth2/sign_in - the login page, which also doubles as a sign out page (it clears cookies)

154155* /oauth2/start - a URL that will redirect to start the OAuth cycle

oauthproxy.go+11

@@ -21,6 +21,7 @@ import (

2121 "github.com/bitly/google_auth_proxy/providers"

2222)

2323

24+const robotsPath = "/robots.txt"

2425const pingPath = "/ping"

2526const signInPath = "/oauth2/sign_in"

2627const oauthStartPath = "/oauth2/start"

@@ -270,6 +271,11 @@ func (p *OauthProxy) SetCookie(rw http.ResponseWriter, req *http.Request, val st

270271 http.SetCookie(rw, cookie)

271272}

272273

274+func (p *OauthProxy) RobotsTxt(rw http.ResponseWriter) {

275+ rw.WriteHeader(http.StatusOK)

276+ fmt.Fprintf(rw, "User-agent: *\nDisallow: /")

277+}

278+

273279func (p *OauthProxy) PingPage(rw http.ResponseWriter) {

274280 rw.WriteHeader(http.StatusOK)

275281 fmt.Fprintf(rw, "OK")

@@ -358,6 +364,11 @@ func (p *OauthProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {

358364 var email string

359365 var access_token string

360366

367+ if req.URL.Path == robotsPath {

368+ p.RobotsTxt(rw)

369+ return

370+ }

371+

361372 if req.URL.Path == pingPath {

362373 p.PingPage(rw)

363374 return

oauthproxy_test.go+16

@@ -67,6 +67,22 @@ func TestEncodedSlashes(t *testing.T) {

6767 }

6868}

6969

70+func TestRobotsTxt(t *testing.T) {

71+ opts := NewOptions()

72+ opts.Upstreams = append(opts.Upstreams, "unused")

73+ opts.ClientID = "bazquux"

74+ opts.ClientSecret = "foobar"

75+ opts.CookieSecret = "xyzzyplugh"

76+ opts.Validate()

77+

78+ proxy := NewOauthProxy(opts, func(string) bool { return true })

79+ rw := httptest.NewRecorder()

80+ req, _ := http.NewRequest("GET", "/robots.txt", nil)

81+ proxy.ServeHTTP(rw, req)

82+ assert.Equal(t, 200, rw.Code)

83+ assert.Equal(t, "User-agent: *\nDisallow: /", rw.Body.String())

84+}

85+

7086type TestProvider struct {

7187 *providers.ProviderData

7288 EmailAddress string