git.iko.soy/typeable/oauth2-proxy
on master help sign in
Merge pull request #1104 from oauth2-proxy/robots-page

Allow custom robots text pages

commit455d649165fac50cd30fdd574ffb77792a76d129 authorJoel Speed <Joel.speed@hotmail.co.uk> dateSun 21 Mar 2021, 20:02 (6 years ago) parentc1267bb0615a6a
7 files changed+284−8
CHANGELOG.md+1 oauthproxy.go+1−7 oauthproxy_test.go+1−1 pkg/app/pagewriter/pagewriter.go+8 pkg/app/pagewriter/robots.txt+2 pkg/app/pagewriter/static_pages.go+118 pkg/app/pagewriter/static_pages_test.go+153
CHANGELOG.md+1
@@ -12,6 +12,7 @@
1212
1313## Changes since v7.0.1
1414
15+- [#1104](https://github.com/oauth2-proxy/oauth2-proxy/pull/1104) Allow custom robots text pages (@JoelSpeed)
1516- [#1045](https://github.com/oauth2-proxy/oauth2-proxy/pull/1045) Ensure redirect URI always has a scheme (@JoelSpeed)
1617- [#1103](https://github.com/oauth2-proxy/oauth2-proxy/pull/1103) Deprecate upstream request signatures (@NickMeves)
1718- [1087](https://github.com/oauth2-proxy/oauth2-proxy/pull/1087) Support Request ID in logging (@NickMeves)
oauthproxy.go+1−7
@@ -577,13 +577,7 @@ func (p *OAuthProxy) serveHTTP(rw http.ResponseWriter, req *http.Request) {
577577
578578// RobotsTxt disallows scraping pages from the OAuthProxy
579579func (p *OAuthProxy) RobotsTxt(rw http.ResponseWriter, req *http.Request) {
580− _, err := fmt.Fprintf(rw, "User-agent: *\nDisallow: /")
581− if err != nil {
582− logger.Printf("Error writing robots.txt: %v", err)
583− p.ErrorPage(rw, req, http.StatusInternalServerError, err.Error())
584− return
585− }
586− rw.WriteHeader(http.StatusOK)
580+ p.pageWriter.WriteRobotsTxt(rw, req)
587581}
588582
589583// ErrorPage writes an error response
oauthproxy_test.go+1−1
@@ -57,7 +57,7 @@ func TestRobotsTxt(t *testing.T) {
5757 req, _ := http.NewRequest("GET", "/robots.txt", nil)
5858 proxy.ServeHTTP(rw, req)
5959 assert.Equal(t, 200, rw.Code)
60− assert.Equal(t, "User-agent: *\nDisallow: /", rw.Body.String())
60+ assert.Equal(t, "User-agent: *\nDisallow: /\n", rw.Body.String())
6161}
6262
6363func TestIsValidRedirect(t *testing.T) {
pkg/app/pagewriter/pagewriter.go+8
@@ -13,12 +13,14 @@ type Writer interface {
1313 WriteSignInPage(rw http.ResponseWriter, req *http.Request, redirectURL string)
1414 WriteErrorPage(rw http.ResponseWriter, opts ErrorPageOpts)
1515 ProxyErrorHandler(rw http.ResponseWriter, req *http.Request, proxyErr error)
16+ WriteRobotsTxt(rw http.ResponseWriter, req *http.Request)
1617}
1718
1819// pageWriter implements the Writer interface
1920type pageWriter struct {
2021 *errorPageWriter
2122 *signInPageWriter
23+ *staticPageWriter
2224}
2325
2426// Opts contains all options required to configure the template
@@ -88,8 +90,14 @@ func NewWriter(opts Opts) (Writer, error) {
8890 logoData: logoData,
8991 }
9092
93+ staticPages, err := newStaticPageWriter(opts.TemplatesPath, errorPage)
94+ if err != nil {
95+ return nil, fmt.Errorf("error loading static page writer: %v", err)
96+ }
97+
9198 return &pageWriter{
9299 errorPageWriter: errorPage,
93100 signInPageWriter: signInPage,
101+ staticPageWriter: staticPages,
94102 }, nil
95103}
pkg/app/pagewriter/robots.txtnew file+2
@@ -0,0 +1,2 @@
1+User-agent: *
2+Disallow: /
pkg/app/pagewriter/static_pages.gonew file+118
@@ -0,0 +1,118 @@
1+package pagewriter
2+
3+import (
4+ // Import embed to allow importing default page templates
5+ _ "embed"
6+ "fmt"
7+ "net/http"
8+ "os"
9+ "path/filepath"
10+
11+ middlewareapi "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/apis/middleware"
12+ "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/logger"
13+)
14+
15+const (
16+ robotsTxtName = "robots.txt"
17+)
18+
19+//go:embed robots.txt
20+var defaultRobotsTxt []byte
21+
22+// staticPageWriter is used to write static pages.
23+type staticPageWriter struct {
24+ pageGetter *pageGetter
25+ errorPageWriter *errorPageWriter
26+}
27+
28+// WriteRobotsTxt writes the robots.txt content to the response writer.
29+func (s *staticPageWriter) WriteRobotsTxt(rw http.ResponseWriter, req *http.Request) {
30+ s.writePage(rw, req, robotsTxtName)
31+}
32+
33+// writePage writes the content of the page to the response writer.
34+func (s *staticPageWriter) writePage(rw http.ResponseWriter, req *http.Request, pageName string) {
35+ _, err := rw.Write(s.pageGetter.getPage(pageName))
36+ if err != nil {
37+ logger.Printf("Error writing %q: %v", pageName, err)
38+ scope := middlewareapi.GetRequestScope(req)
39+ s.errorPageWriter.WriteErrorPage(rw, ErrorPageOpts{
40+ Status: http.StatusInternalServerError,
41+ RequestID: scope.RequestID,
42+ AppError: err.Error(),
43+ })
44+ return
45+ }
46+}
47+
48+func newStaticPageWriter(customDir string, errorWriter *errorPageWriter) (*staticPageWriter, error) {
49+ pageGetter, err := loadStaticPages(customDir)
50+ if err != nil {
51+ return nil, fmt.Errorf("could not load static pages: %v", err)
52+ }
53+
54+ return &staticPageWriter{
55+ pageGetter: pageGetter,
56+ errorPageWriter: errorWriter,
57+ }, nil
58+}
59+
60+// loadStaticPages loads static page content from the custom directory provided.
61+// If any file is not provided in the custom directory, the default will be used
62+// instead.
63+// Statis files include:
64+// - robots.txt
65+func loadStaticPages(customDir string) (*pageGetter, error) {
66+ pages := newPageGetter(customDir)
67+
68+ if err := pages.addPage(robotsTxtName, defaultRobotsTxt); err != nil {
69+ return nil, fmt.Errorf("could not add robots.txt: %v", err)
70+ }
71+
72+ return pages, nil
73+}
74+
75+// pageGetter is used to load and read page content for static pages.
76+type pageGetter struct {
77+ pages map[string][]byte
78+ dir string
79+}
80+
81+// newPageGetter creates a new page getter for the custom directory.
82+func newPageGetter(customDir string) *pageGetter {
83+ return &pageGetter{
84+ pages: make(map[string][]byte),
85+ dir: customDir,
86+ }
87+}
88+
89+// addPage loads a new page into the pageGetter.
90+// If the given file name does not exist in the custom directory, the default
91+// content will be used instead.
92+func (p *pageGetter) addPage(fileName string, defaultContent []byte) error {
93+ filePath := filepath.Join(p.dir, fileName)
94+ if p.dir != "" && isFile(filePath) {
95+ content, err := os.ReadFile(filePath)
96+ if err != nil {
97+ return fmt.Errorf("could not read file: %v", err)
98+ }
99+
100+ p.pages[fileName] = content
101+ return nil
102+ }
103+
104+ // No custom content defined, use the default.
105+ p.pages[fileName] = defaultContent
106+ return nil
107+}
108+
109+// getPage returns the page content for a given page.
110+func (p *pageGetter) getPage(name string) []byte {
111+ content, ok := p.pages[name]
112+ if !ok {
113+ // If the page isn't registered, something went wrong and there is a bug.
114+ // Tests should make sure this code path is never hit.
115+ panic(fmt.Sprintf("Static page %q not found", name))
116+ }
117+ return content
118+}
pkg/app/pagewriter/static_pages_test.gonew file+153
@@ -0,0 +1,153 @@
1+package pagewriter
2+
3+import (
4+ "errors"
5+ "html/template"
6+ "io/ioutil"
7+ "net/http"
8+ "net/http/httptest"
9+ "os"
10+ "path/filepath"
11+
12+ middlewareapi "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/apis/middleware"
13+ . "github.com/onsi/ginkgo"
14+ . "github.com/onsi/gomega"
15+)
16+
17+var _ = Describe("Static Pages", func() {
18+ var customDir string
19+ const customRobots = "User-agent: *\nAllow: /\n"
20+ var errorPage *errorPageWriter
21+ var request *http.Request
22+
23+ BeforeEach(func() {
24+ errorTmpl, err := template.New("").Parse("{{.Title}}")
25+ Expect(err).ToNot(HaveOccurred())
26+ errorPage = &errorPageWriter{
27+ template: errorTmpl,
28+ }
29+
30+ customDir, err = ioutil.TempDir("", "oauth2-proxy-static-pages-test")
31+ Expect(err).ToNot(HaveOccurred())
32+
33+ robotsTxtFile := filepath.Join(customDir, robotsTxtName)
34+ Expect(ioutil.WriteFile(robotsTxtFile, []byte(customRobots), 0400)).To(Succeed())
35+
36+ request = httptest.NewRequest("", "http://127.0.0.1/", nil)
37+ request = middlewareapi.AddRequestScope(request, &middlewareapi.RequestScope{
38+ RequestID: testRequestID,
39+ })
40+ })
41+
42+ AfterEach(func() {
43+ Expect(os.RemoveAll(customDir)).To(Succeed())
44+ })
45+
46+ Context("Static Page Writer", func() {
47+ Context("With custom content", func() {
48+ var pageWriter *staticPageWriter
49+
50+ BeforeEach(func() {
51+ var err error
52+ pageWriter, err = newStaticPageWriter(customDir, errorPage)
53+ Expect(err).ToNot(HaveOccurred())
54+ })
55+
56+ Context("WriterRobotsTxt", func() {
57+ It("Should write the custom robots txt", func() {
58+ recorder := httptest.NewRecorder()
59+ pageWriter.WriteRobotsTxt(recorder, request)
60+
61+ body, err := ioutil.ReadAll(recorder.Result().Body)
62+ Expect(err).ToNot(HaveOccurred())
63+ Expect(string(body)).To(Equal(customRobots))
64+
65+ Expect(recorder.Result().StatusCode).To(Equal(http.StatusOK))
66+ })
67+ })
68+ })
69+
70+ Context("Without custom content", func() {
71+ var pageWriter *staticPageWriter
72+
73+ BeforeEach(func() {
74+ var err error
75+ pageWriter, err = newStaticPageWriter("", errorPage)
76+ Expect(err).ToNot(HaveOccurred())
77+ })
78+
79+ Context("WriterRobotsTxt", func() {
80+ It("Should write the custom robots txt", func() {
81+ recorder := httptest.NewRecorder()
82+ pageWriter.WriteRobotsTxt(recorder, request)
83+
84+ body, err := ioutil.ReadAll(recorder.Result().Body)
85+ Expect(err).ToNot(HaveOccurred())
86+ Expect(string(body)).To(Equal(string(defaultRobotsTxt)))
87+
88+ Expect(recorder.Result().StatusCode).To(Equal(http.StatusOK))
89+ })
90+
91+ It("Should serve an error if it cannot write the page", func() {
92+ recorder := &testBadResponseWriter{
93+ ResponseRecorder: httptest.NewRecorder(),
94+ }
95+ pageWriter.WriteRobotsTxt(recorder, request)
96+
97+ body, err := ioutil.ReadAll(recorder.Result().Body)
98+ Expect(err).ToNot(HaveOccurred())
99+ Expect(string(body)).To(Equal(string("Internal Server Error")))
100+
101+ Expect(recorder.Result().StatusCode).To(Equal(http.StatusInternalServerError))
102+ })
103+ })
104+ })
105+ })
106+
107+ Context("loadStaticPages", func() {
108+ Context("With custom content", func() {
109+ Context("And a custom robots txt", func() {
110+ It("Loads the custom content", func() {
111+ pages, err := loadStaticPages(customDir)
112+ Expect(err).ToNot(HaveOccurred())
113+ Expect(pages.pages).To(HaveLen(1))
114+ Expect(pages.getPage(robotsTxtName)).To(BeEquivalentTo(customRobots))
115+ })
116+ })
117+
118+ Context("And no custom robots txt", func() {
119+ It("returns the default content", func() {
120+ robotsTxtFile := filepath.Join(customDir, robotsTxtName)
121+ Expect(os.Remove(robotsTxtFile)).To(Succeed())
122+
123+ pages, err := loadStaticPages(customDir)
124+ Expect(err).ToNot(HaveOccurred())
125+ Expect(pages.pages).To(HaveLen(1))
126+ Expect(pages.getPage(robotsTxtName)).To(BeEquivalentTo(defaultRobotsTxt))
127+ })
128+ })
129+ })
130+
131+ Context("Without custom content", func() {
132+ It("Loads the default content", func() {
133+ pages, err := loadStaticPages("")
134+ Expect(err).ToNot(HaveOccurred())
135+ Expect(pages.pages).To(HaveLen(1))
136+ Expect(pages.getPage(robotsTxtName)).To(BeEquivalentTo(defaultRobotsTxt))
137+ })
138+ })
139+ })
140+})
141+
142+type testBadResponseWriter struct {
143+ *httptest.ResponseRecorder
144+ firstWriteCalled bool
145+}
146+
147+func (b *testBadResponseWriter) Write(buf []byte) (int, error) {
148+ if !b.firstWriteCalled {
149+ b.firstWriteCalled = true
150+ return 0, errors.New("write closed")
151+ }
152+ return b.ResponseRecorder.Write(buf)
153+}