git.iko.soy/typeable/oauth2-proxy
on master help sign in
Merge pull request #274 from toshi-miura/ap-gh-pagination-with-lastpage

Added pagination support for /user/teams github api with link header .

commit9fdb8acec6d45e91d2861b5fc4b4885bae5e770b authorJoel Speed <Joel.speed@hotmail.co.uk> dateThu 14 Nov 2019, 14:26 (7 years ago) parent9f920b06326660
3 files changed+86−29
CHANGELOG.md+2 providers/github.go+83−28 providers/github_test.go+1−1
CHANGELOG.md+2
@@ -9,10 +9,12 @@
99- [#258](https://github.com/pusher/oauth2_proxy/pull/258) Add IDToken for Azure provider
1010 - This PR adds the IDToken into the session for the Azure provider allowing requests to a backend to be identified as a specific user. As a consequence, if you are using a cookie to store the session the cookie will now exceed the 4kb size limit and be split into multiple cookies. This can cause problems when using nginx as a proxy, resulting in no cookie being passed at all. Either increase the proxy_buffer_size in nginx or implement the redis session storage (see https://pusher.github.io/oauth2_proxy/configuration#redis-storage)
1111- [#286](https://github.com/pusher/oauth2_proxy/pull/286) Requests.go updated with useful error messages (@biotom)
12+- [#274](https://github.com/pusher/oauth2_proxy/pull/274) Supports many github teams with api pagination support (@toshi-miura, @apratina)
1213- [#302](https://github.com/pusher/oauth2_proxy/pull/302) Rewrite dist script (@syscll)
1314- [#304](https://github.com/pusher/oauth2_proxy/pull/304) Add new Logo! :tada: (@JoelSpeed)
1415- [#300](https://github.com/pusher/oauth2_proxy/pull/300) Added userinfo endpoint (@kbabuadze)
1516- [#309](https://github.com/pusher/oauth2_proxy/pull/309) Added support for custom CA when connecting to Redis cache
17+
1618# v4.0.0
1719
1820- [#248](https://github.com/pusher/oauth2_proxy/pull/248) Fix issue with X-Auth-Request-Redirect header being ignored
providers/github.go+83−28
@@ -7,6 +7,7 @@ import (
77 "net/http"
88 "net/url"
99 "path"
10+ "regexp"
1011 "strconv"
1112 "strings"
1213
@@ -75,8 +76,8 @@ func (p *GitHubProvider) hasOrg(accessToken string) (bool, error) {
7576 pn := 1
7677 for {
7778 params := url.Values{
78− "limit": {"200"},
79− "page": {strconv.Itoa(pn)},
79+ "per_page": {"100"},
80+ "page": {strconv.Itoa(pn)},
8081 }
8182
8283 endpoint := &url.URL{
@@ -139,36 +140,90 @@ func (p *GitHubProvider) hasOrgAndTeam(accessToken string) (bool, error) {
139140 } `json:"organization"`
140141 }
141142
142− params := url.Values{
143− "limit": {"200"},
143+ type teamsPage []struct {
144+ Name string `json:"name"`
145+ Slug string `json:"slug"`
146+ Org struct {
147+ Login string `json:"login"`
148+ } `json:"organization"`
144149 }
145150
146− endpoint := &url.URL{
147− Scheme: p.ValidateURL.Scheme,
148− Host: p.ValidateURL.Host,
149− Path: path.Join(p.ValidateURL.Path, "/user/teams"),
150− RawQuery: params.Encode(),
151− }
152− req, _ := http.NewRequest("GET", endpoint.String(), nil)
153− req.Header.Set("Accept", "application/vnd.github.v3+json")
154− req.Header.Set("Authorization", fmt.Sprintf("token %s", accessToken))
155− resp, err := http.DefaultClient.Do(req)
156− if err != nil {
157− return false, err
158− }
151+ pn := 1
152+ last := 0
153+ for {
154+ params := url.Values{
155+ "per_page": {"100"},
156+ "page": {strconv.Itoa(pn)},
157+ }
159158
160− body, err := ioutil.ReadAll(resp.Body)
161− resp.Body.Close()
162− if err != nil {
163− return false, err
164− }
165− if resp.StatusCode != 200 {
166− return false, fmt.Errorf(
167− "got %d from %q %s", resp.StatusCode, endpoint.String(), body)
168− }
159+ endpoint := &url.URL{
160+ Scheme: p.ValidateURL.Scheme,
161+ Host: p.ValidateURL.Host,
162+ Path: path.Join(p.ValidateURL.Path, "/user/teams"),
163+ RawQuery: params.Encode(),
164+ }
169165
170− if err := json.Unmarshal(body, &teams); err != nil {
171− return false, fmt.Errorf("%s unmarshaling %s", err, body)
166+ req, _ := http.NewRequest("GET", endpoint.String(), nil)
167+ req.Header.Set("Accept", "application/vnd.github.v3+json")
168+ req.Header.Set("Authorization", fmt.Sprintf("token %s", accessToken))
169+ resp, err := http.DefaultClient.Do(req)
170+ if err != nil {
171+ return false, err
172+ }
173+
174+ if last == 0 {
175+ // link header may not be obtained
176+ // When paging is not required and all data can be retrieved with a single call
177+
178+ // Conditions for obtaining the link header.
179+ // 1. When paging is required (Example: When the data size is 100 and the page size is 99 or less)
180+ // 2. When it exceeds the paging frame (Example: When there is only 10 records but the second page is called with a page size of 100)
181+
182+ // link header at not last page
183+ // <https://api.github.com/user/teams?page=1&per_page=100>; rel="prev", <https://api.github.com/user/teams?page=1&per_page=100>; rel="last", <https://api.github.com/user/teams?page=1&per_page=100>; rel="first"
184+ // link header at last page (doesn't exist last info)
185+ // <https://api.github.com/user/teams?page=3&per_page=10>; rel="prev", <https://api.github.com/user/teams?page=1&per_page=10>; rel="first"
186+
187+ link := resp.Header.Get("Link")
188+ rep1 := regexp.MustCompile(`(?s).*\<https://api.github.com/user/teams\?page=(.)&per_page=[0-9]+\>; rel="last".*`)
189+ i, converr := strconv.Atoi(rep1.ReplaceAllString(link, "$1"))
190+
191+ // If the last page cannot be taken from the link in the http header, the last variable remains zero
192+ if converr == nil {
193+ last = i
194+ }
195+ }
196+
197+ body, err := ioutil.ReadAll(resp.Body)
198+ if err != nil {
199+ resp.Body.Close()
200+ return false, err
201+ }
202+ resp.Body.Close()
203+
204+ if resp.StatusCode != 200 {
205+ return false, fmt.Errorf(
206+ "got %d from %q %s", resp.StatusCode, endpoint.String(), body)
207+ }
208+
209+ var tp teamsPage
210+ if err := json.Unmarshal(body, &tp); err != nil {
211+ return false, fmt.Errorf("%s unmarshaling %s", err, body)
212+ }
213+ if len(tp) == 0 {
214+ break
215+ }
216+
217+ teams = append(teams, tp...)
218+
219+ if pn == last {
220+ break
221+ }
222+ if last == 0 {
223+ break
224+ }
225+
226+ pn++
172227 }
173228
174229 var hasOrg bool
providers/github_test.go+1−1
@@ -32,7 +32,7 @@ func testGitHubBackend(payload []string) *httptest.Server {
3232 pathToQueryMap := map[string][]string{
3333 "/user": {""},
3434 "/user/emails": {""},
35− "/user/orgs": {"limit=200&page=1", "limit=200&page=2", "limit=200&page=3"},
35+ "/user/orgs": {"page=1&per_page=100", "page=2&per_page=100", "page=3&per_page=100"},
3636 }
3737
3838 return httptest.NewServer(http.HandlerFunc(