// Copyright 2019 The Gitea Authors. All rights reserved.// Copyright 2018 Jonas Franz. All rights reserved.// SPDX-License-Identifier: MIT package migrations import (	"context"	"fmt"	"io"	"net/http"	"net/url"	"strconv"	"strings"	"time" 	"forgejo.org/modules/git"	"forgejo.org/modules/log"	base "forgejo.org/modules/migration"	"forgejo.org/modules/proxy"	"forgejo.org/modules/structs" 	"github.com/google/go-github/v64/github"	"golang.org/x/oauth2") var (	_ base.Downloader        = &GithubDownloaderV3{}	_ base.DownloaderFactory = &GithubDownloaderV3Factory{}	// GithubLimitRateRemaining limit to wait for new rate to apply	GithubLimitRateRemaining = 0) func init() {	RegisterDownloaderFactory(&GithubDownloaderV3Factory{})} // GithubDownloaderV3Factory defines a github downloader v3 factorytype GithubDownloaderV3Factory struct{} // New returns a Downloader related to this factory according MigrateOptionsfunc (f *GithubDownloaderV3Factory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {	u, err := url.Parse(opts.CloneAddr)	if err != nil {		return nil, err	} 	// some users are using the github redirect url for migration	if u.Host == "www.github.com" {		u.Host = "github.com"	} 	baseURL := u.Scheme + "://" + u.Host	fields := strings.Split(u.Path, "/")	oldOwner := fields[1]	oldName := strings.TrimSuffix(fields[2], ".git") 	log.Trace("Create github downloader BaseURL: %s %s/%s", baseURL, oldOwner, oldName) 	return NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil} // GitServiceType returns the type of git servicefunc (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {	return structs.GithubService} // GithubDownloaderV3 implements a Downloader interface to get repository information// from github via APIv3type GithubDownloaderV3 struct {	base.NullDownloader	ctx           context.Context	clients       []*github.Client	baseURL       string	repoOwner     string	repoName      string	userName      string	password      string	rates         []*github.Rate	curClientIdx  int	maxPerPage    int	SkipReactions bool	SkipReviews   bool} // NewGithubDownloaderV3 creates a github Downloader via github v3 APIfunc NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {	downloader := GithubDownloaderV3{		userName:   userName,		baseURL:    baseURL,		password:   password,		ctx:        ctx,		repoOwner:  repoOwner,		repoName:   repoName,		maxPerPage: 100,	} 	if token != "" {		tokens := strings.Split(token, ",")		for _, token := range tokens {			token = strings.TrimSpace(token)			ts := oauth2.StaticTokenSource(				&oauth2.Token{AccessToken: token},			)			client := &http.Client{				Transport: &oauth2.Transport{					Base:   NewMigrationHTTPTransport(),					Source: oauth2.ReuseTokenSource(nil, ts),				},			} 			downloader.addClient(client, baseURL)		}	} else {		transport := NewMigrationHTTPTransport()		transport.Proxy = func(req *http.Request) (*url.URL, error) {			req.SetBasicAuth(userName, password)			return proxy.Proxy()(req)		}		client := &http.Client{			Transport: transport,		}		downloader.addClient(client, baseURL)	}	return &downloader} // String implements Stringerfunc (g *GithubDownloaderV3) String() string {	return fmt.Sprintf("migration from github server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)} func (g *GithubDownloaderV3) LogString() string {	if g == nil {		return "<GithubDownloaderV3 nil>"	}	return fmt.Sprintf("<GithubDownloaderV3 %s %s/%s>", g.baseURL, g.repoOwner, g.repoName)} func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) {	githubClient := github.NewClient(client)	if baseURL != "https://github.com" {		githubClient, _ = githubClient.WithEnterpriseURLs(baseURL, baseURL)	}	g.clients = append(g.clients, githubClient)	g.rates = append(g.rates, nil)} // SetContext set contextfunc (g *GithubDownloaderV3) SetContext(ctx context.Context) {	g.ctx = ctx} func (g *GithubDownloaderV3) waitAndPickClient() {	var recentIdx int	var maxRemaining int	for i := 0; i < len(g.clients); i++ {		if g.rates[i] != nil && g.rates[i].Remaining > maxRemaining {			maxRemaining = g.rates[i].Remaining			recentIdx = i		}	}	g.curClientIdx = recentIdx // if no max remain, it will always pick the first client. 	for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining {		timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time))		select {		case <-g.ctx.Done():			timer.Stop()			return		case <-timer.C:		} 		err := g.RefreshRate()		if err != nil {			log.Error("g.getClient().RateLimit.Get: %s", err)		}	}} // RefreshRate update the current rate (doesn't count in rate limit)func (g *GithubDownloaderV3) RefreshRate() error {	rates, _, err := g.getClient().RateLimit.Get(g.ctx)	if err != nil {		// if rate limit is not enabled, ignore it		if strings.Contains(err.Error(), "404") {			g.setRate(nil)			return nil		}		return err	} 	g.setRate(rates.GetCore())	return nil} func (g *GithubDownloaderV3) getClient() *github.Client {	return g.clients[g.curClientIdx]} func (g *GithubDownloaderV3) setRate(rate *github.Rate) {	g.rates[g.curClientIdx] = rate} // GetRepoInfo returns a repository informationfunc (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {	g.waitAndPickClient()	gr, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)	if err != nil {		return nil, err	}	g.setRate(&resp.Rate) 	// convert github repo to stand Repo	return &base.Repository{		Owner:         g.repoOwner,		Name:          gr.GetName(),		IsPrivate:     gr.GetPrivate(),		Description:   gr.GetDescription(),		OriginalURL:   gr.GetHTMLURL(),		CloneURL:      gr.GetCloneURL(),		DefaultBranch: gr.GetDefaultBranch(),		Website:       gr.GetHomepage(),	}, nil} // GetTopics return github topicsfunc (g *GithubDownloaderV3) GetTopics() ([]string, error) {	g.waitAndPickClient()	r, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)	if err != nil {		return nil, err	}	g.setRate(&resp.Rate)	return r.Topics, nil} // GetMilestones returns milestonesfunc (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {	perPage := g.maxPerPage	milestones := make([]*base.Milestone, 0, perPage)	for i := 1; ; i++ {		g.waitAndPickClient()		ms, resp, err := g.getClient().Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,			&github.MilestoneListOptions{				State: "all",				ListOptions: github.ListOptions{					Page:    i,					PerPage: perPage,				},			})		if err != nil {			return nil, err		}		g.setRate(&resp.Rate) 		for _, m := range ms {			state := "open"			if m.State != nil {				state = *m.State			}			milestones = append(milestones, &base.Milestone{				Title:       m.GetTitle(),				Description: m.GetDescription(),				Deadline:    m.DueOn.GetTime(),				State:       state,				Created:     m.GetCreatedAt().Time,				Updated:     m.UpdatedAt.GetTime(),				Closed:      m.ClosedAt.GetTime(),			})		}		if len(ms) < perPage {			break		}	}	return milestones, nil} func convertGithubLabel(label *github.Label) *base.Label {	return &base.Label{		Name:        label.GetName(),		Color:       label.GetColor(),		Description: label.GetDescription(),	}} // GetLabels returns labelsfunc (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {	perPage := g.maxPerPage	labels := make([]*base.Label, 0, perPage)	for i := 1; ; i++ {		g.waitAndPickClient()		ls, resp, err := g.getClient().Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,			&github.ListOptions{				Page:    i,				PerPage: perPage,			})		if err != nil {			return nil, err		}		g.setRate(&resp.Rate) 		for _, label := range ls {			labels = append(labels, convertGithubLabel(label))		}		if len(ls) < perPage {			break		}	}	return labels, nil} func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) *base.Release {	// GitHub allows committish to be a reference.	// In this case, we need to remove the prefix, i.e. convert "refs/heads/main" to "main".	targetCommitish := strings.TrimPrefix(rel.GetTargetCommitish(), git.BranchPrefix) 	r := &base.Release{		Name:            rel.GetName(),		TagName:         rel.GetTagName(),		TargetCommitish: targetCommitish,		Draft:           rel.GetDraft(),		Prerelease:      rel.GetPrerelease(),		Created:         rel.GetCreatedAt().Time,		PublisherID:     rel.GetAuthor().GetID(),		PublisherName:   rel.GetAuthor().GetLogin(),		PublisherEmail:  rel.GetAuthor().GetEmail(),		Body:            rel.GetBody(),	} 	if rel.PublishedAt != nil {		r.Published = rel.PublishedAt.Time	} 	httpClient := NewMigrationHTTPClient() 	for _, asset := range rel.Assets {		assetID := *asset.ID // Don't optimize this, for closure we need a local variable		r.Assets = append(r.Assets, &base.ReleaseAsset{			ID:            asset.GetID(),			Name:          asset.GetName(),			ContentType:   asset.ContentType,			Size:          asset.Size,			DownloadCount: asset.DownloadCount,			Created:       asset.CreatedAt.Time,			Updated:       asset.UpdatedAt.Time,			DownloadFunc: func() (io.ReadCloser, error) {				g.waitAndPickClient()				readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil)				if err != nil {					return nil, err				}				if err := g.RefreshRate(); err != nil {					log.Error("g.getClient().RateLimits: %s", err)				} 				if readCloser != nil {					return readCloser, nil				} 				if redirectURL == "" {					return nil, fmt.Errorf("no release asset found for %d", assetID)				} 				// Prevent open redirect				if !hasBaseURL(redirectURL, g.baseURL) &&					!hasBaseURL(redirectURL, "https://objects.githubusercontent.com/") &&					!hasBaseURL(redirectURL, "https://release-assets.githubusercontent.com/") {					WarnAndNotice("Unexpected AssetURL for assetID[%d] in %s: %s", asset.GetID(), g, redirectURL) 					return io.NopCloser(strings.NewReader(redirectURL)), nil				} 				g.waitAndPickClient()				req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil)				if err != nil {					return nil, err				}				resp, err := httpClient.Do(req)				err1 := g.RefreshRate()				if err1 != nil {					log.Error("g.RefreshRate(): %s", err1)				}				if err != nil {					return nil, err				}				return resp.Body, nil			},		})	}	return r} // GetReleases returns releasesfunc (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {	perPage := g.maxPerPage	releases := make([]*base.Release, 0, perPage)	for i := 1; ; i++ {		g.waitAndPickClient()		ls, resp, err := g.getClient().Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,			&github.ListOptions{				Page:    i,				PerPage: perPage,			})		if err != nil {			return nil, err		}		g.setRate(&resp.Rate) 		for _, release := range ls {			releases = append(releases, g.convertGithubRelease(release))		}		if len(ls) < perPage {			break		}	}	return releases, nil} // GetIssues returns issues according start and limitfunc (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {	if perPage > g.maxPerPage {		perPage = g.maxPerPage	}	opt := &github.IssueListByRepoOptions{		Sort:      "created",		Direction: "asc",		State:     "all",		ListOptions: github.ListOptions{			PerPage: perPage,			Page:    page,		},	} 	allIssues := make([]*base.Issue, 0, perPage)	g.waitAndPickClient()	issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)	if err != nil {		return nil, false, fmt.Errorf("error while listing repos: %w", err)	}	log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues))	g.setRate(&resp.Rate)	for _, issue := range issues {		if issue.IsPullRequest() {			continue		} 		labels := make([]*base.Label, 0, len(issue.Labels))		for _, l := range issue.Labels {			labels = append(labels, convertGithubLabel(l))		} 		// get reactions		var reactions []*base.Reaction		if !g.SkipReactions {			for i := 1; ; i++ {				g.waitAndPickClient()				res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{					Page:    i,					PerPage: perPage,				})				if err != nil {					return nil, false, err				}				g.setRate(&resp.Rate)				if len(res) == 0 {					break				}				for _, reaction := range res {					reactions = append(reactions, &base.Reaction{						UserID:   reaction.User.GetID(),						UserName: reaction.User.GetLogin(),						Content:  reaction.GetContent(),					})				}			}		} 		var assignees []string		for i := range issue.Assignees {			assignees = append(assignees, issue.Assignees[i].GetLogin())		} 		allIssues = append(allIssues, &base.Issue{			Title:        *issue.Title,			Number:       int64(*issue.Number),			PosterID:     issue.GetUser().GetID(),			PosterName:   issue.GetUser().GetLogin(),			PosterEmail:  issue.GetUser().GetEmail(),			Content:      issue.GetBody(),			Milestone:    issue.GetMilestone().GetTitle(),			State:        issue.GetState(),			Created:      issue.GetCreatedAt().Time,			Updated:      issue.GetUpdatedAt().Time,			Labels:       labels,			Reactions:    reactions,			Closed:       issue.ClosedAt.GetTime(),			IsLocked:     issue.GetLocked(),			Assignees:    assignees,			ForeignIndex: int64(*issue.Number),

Lines 1–500 of 903. Load more