// Copyright 2019 The Gitea Authors. All rights reserved.// SPDX-License-Identifier: MIT package migrations import (	"context"	"errors"	"fmt"	"io"	"net/http"	"net/url"	"path"	"regexp"	"strings"	"time" 	issues_model "forgejo.org/models/issues"	user_model "forgejo.org/models/user"	"forgejo.org/modules/container"	"forgejo.org/modules/log"	base "forgejo.org/modules/migration"	"forgejo.org/modules/structs" 	gitlab "gitlab.com/gitlab-org/api/client-go") var (	_ base.Downloader        = &GitlabDownloader{}	_ base.DownloaderFactory = &GitlabDownloaderFactory{}) func init() {	RegisterDownloaderFactory(&GitlabDownloaderFactory{})} // GitlabDownloaderFactory defines a gitlab downloader factorytype GitlabDownloaderFactory struct{} // New returns a Downloader related to this factory according MigrateOptionsfunc (f *GitlabDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {	u, err := url.Parse(opts.CloneAddr)	if err != nil {		return nil, err	} 	baseURL := u.Scheme + "://" + u.Host	repoNameSpace := strings.TrimPrefix(u.Path, "/")	repoNameSpace = strings.TrimSuffix(repoNameSpace, ".git") 	log.Trace("Create gitlab downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace) 	return NewGitlabDownloader(ctx, baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword, opts.AuthToken)} // GitServiceType returns the type of git servicefunc (f *GitlabDownloaderFactory) GitServiceType() structs.GitServiceType {	return structs.GitlabService} type gitlabIIDResolver struct {	maxIssueIID int64	frozen      bool} func (r *gitlabIIDResolver) recordIssueIID(issueIID int) {	if r.frozen {		panic("cannot record issue IID after pull request IID generation has started")	}	r.maxIssueIID = max(r.maxIssueIID, int64(issueIID))} func (r *gitlabIIDResolver) generatePullRequestNumber(mrIID int) int64 {	r.frozen = true	return r.maxIssueIID + int64(mrIID)} // GitlabDownloader implements a Downloader interface to get repository information// from gitlab via go-gitlab// - issueCount is incremented in GetIssues() to ensure PR and Issue numbers do not overlap,// because Gitlab has individual Issue and Pull Request numbers.type GitlabDownloader struct {	base.NullDownloader	ctx         context.Context	client      *gitlab.Client	baseURL     string	repoID      int	repoName    string	iidResolver gitlabIIDResolver	maxPerPage  int} // NewGitlabDownloader creates a gitlab Downloader via gitlab API////	Use either a username/password, personal token entered into the username field, or anonymous/public access//	Note: Public access only allows very basic accessfunc NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {	gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(NewMigrationHTTPClient()))	// Only use basic auth if token is blank and password is NOT	// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage	if token == "" && password != "" {		gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(NewMigrationHTTPClient()))	} 	if err != nil {		log.Trace("Error logging into gitlab: %v", err)		return nil, err	} 	// split namespace and subdirectory	pathParts := strings.Split(strings.Trim(repoPath, "/"), "/")	var resp *gitlab.Response	u, _ := url.Parse(baseURL)	for len(pathParts) >= 2 {		_, resp, err = gitlabClient.Version.GetVersion()		if err == nil || resp != nil && resp.StatusCode == http.StatusUnauthorized {			err = nil // if no authentication given, this still should work			break		} 		u.Path = path.Join(u.Path, pathParts[0])		baseURL = u.String()		pathParts = pathParts[1:]		_ = gitlab.WithBaseURL(baseURL)(gitlabClient)		repoPath = strings.Join(pathParts, "/")	}	if err != nil {		log.Trace("Error could not get gitlab version: %v", err)		return nil, err	} 	log.Trace("gitlab downloader: use BaseURL: '%s' and RepoPath: '%s'", baseURL, repoPath) 	// Grab and store project/repo ID here, due to issues using the URL escaped path	gr, _, err := gitlabClient.Projects.GetProject(repoPath, nil, nil, gitlab.WithContext(ctx))	if err != nil {		log.Trace("Error retrieving project: %v", err)		return nil, err	} 	if gr == nil {		log.Trace("Error getting project, project is nil")		return nil, errors.New("Error getting project, project is nil")	} 	return &GitlabDownloader{		ctx:        ctx,		client:     gitlabClient,		baseURL:    baseURL,		repoID:     gr.ID,		repoName:   gr.Name,		maxPerPage: 100,	}, nil} // String implements Stringerfunc (g *GitlabDownloader) String() string {	return fmt.Sprintf("migration from gitlab server %s [%d]/%s", g.baseURL, g.repoID, g.repoName)} func (g *GitlabDownloader) LogString() string {	if g == nil {		return "<GitlabDownloader nil>"	}	return fmt.Sprintf("<GitlabDownloader %s [%d]/%s>", g.baseURL, g.repoID, g.repoName)} // SetContext set contextfunc (g *GitlabDownloader) SetContext(ctx context.Context) {	g.ctx = ctx} // GetRepoInfo returns a repository informationfunc (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) {	gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx))	if err != nil {		return nil, err	} 	var private bool	switch gr.Visibility {	case gitlab.InternalVisibility:		private = true	case gitlab.PrivateVisibility:		private = true	} 	var owner string	if gr.Owner == nil {		log.Trace("gr.Owner is nil, trying to get owner from Namespace")		if gr.Namespace != nil && gr.Namespace.Kind == "user" {			owner = gr.Namespace.Path		}	} else {		owner = gr.Owner.Username	} 	// convert gitlab repo to stand Repo	return &base.Repository{		Owner:         owner,		Name:          gr.Name,		IsPrivate:     private,		Description:   gr.Description,		OriginalURL:   gr.WebURL,		CloneURL:      gr.HTTPURLToRepo,		DefaultBranch: gr.DefaultBranch,	}, nil} // GetTopics return gitlab topicsfunc (g *GitlabDownloader) GetTopics() ([]string, error) {	gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx))	if err != nil {		return nil, err	}	return gr.TagList, err} // GetMilestones returns milestonesfunc (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) {	perPage := g.maxPerPage	state := "all"	milestones := make([]*base.Milestone, 0, perPage)	for i := 1; ; i++ {		ms, _, err := g.client.Milestones.ListMilestones(g.repoID, &gitlab.ListMilestonesOptions{			State: &state,			ListOptions: gitlab.ListOptions{				Page:    i,				PerPage: perPage,			},		}, nil, gitlab.WithContext(g.ctx))		if err != nil {			return nil, err		} 		for _, m := range ms {			var desc string			if m.Description != "" {				desc = m.Description			}			state := "open"			var closedAt *time.Time			if m.State != "" {				state = m.State				if state == "closed" {					closedAt = m.UpdatedAt				}			} 			var deadline *time.Time			if m.DueDate != nil {				deadlineParsed, err := time.Parse("2006-01-02", m.DueDate.String())				if err != nil {					log.Trace("Error parsing Milestone DueDate time")					deadline = nil				} else {					deadline = &deadlineParsed				}			} 			milestones = append(milestones, &base.Milestone{				Title:       m.Title,				Description: desc,				Deadline:    deadline,				State:       state,				Created:     *m.CreatedAt,				Updated:     m.UpdatedAt,				Closed:      closedAt,			})		}		if len(ms) < perPage {			break		}	}	return milestones, nil} func (g *GitlabDownloader) normalizeColor(val string) string {	val = strings.TrimLeft(val, "#")	val = strings.ToLower(val)	if len(val) == 3 {		c := []rune(val)		val = fmt.Sprintf("%c%c%c%c%c%c", c[0], c[0], c[1], c[1], c[2], c[2])	}	if len(val) != 6 {		return ""	}	return val} // GetLabels returns labelsfunc (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {	perPage := g.maxPerPage	labels := make([]*base.Label, 0, perPage)	for i := 1; ; i++ {		ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{			Page:    i,			PerPage: perPage,		}}, nil, gitlab.WithContext(g.ctx))		if err != nil {			return nil, err		}		for _, label := range ls {			baseLabel := &base.Label{				Name:        strings.Replace(label.Name, "::", "/", 1),				Color:       g.normalizeColor(label.Color),				Description: label.Description,				Exclusive:   strings.Contains(label.Name, "::"),			}			labels = append(labels, baseLabel)		}		if len(ls) < perPage {			break		}	}	return labels, nil} func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Release {	var zero int	r := &base.Release{		TagName:         rel.TagName,		TargetCommitish: rel.Commit.ID,		Name:            rel.Name,		Body:            rel.Description,		Created:         *rel.CreatedAt,		PublisherID:     int64(rel.Author.ID),		PublisherName:   rel.Author.Username,	} 	httpClient := NewMigrationHTTPClient() 	for k, asset := range rel.Assets.Links {		assetID := asset.ID // Don't optimize this, for closure we need a local variable		r.Assets = append(r.Assets, &base.ReleaseAsset{			ID:            int64(asset.ID),			Name:          asset.Name,			ContentType:   &rel.Assets.Sources[k].Format,			Size:          &zero,			DownloadCount: &zero,			DownloadFunc: func() (io.ReadCloser, error) {				link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, assetID, gitlab.WithContext(g.ctx))				if err != nil {					return nil, err				} 				if !hasBaseURL(link.URL, g.baseURL) {					WarnAndNotice("Unexpected AssetURL for assetID[%d] in %s: %s", assetID, g, link.URL)					return io.NopCloser(strings.NewReader(link.URL)), nil				} 				req, err := http.NewRequest("GET", link.URL, nil)				if err != nil {					return nil, err				}				req = req.WithContext(g.ctx)				resp, err := httpClient.Do(req)				if err != nil {					return nil, err				} 				// resp.Body is closed by the uploader				return resp.Body, nil			},		})	}	return r} // GetReleases returns releasesfunc (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {	perPage := g.maxPerPage	releases := make([]*base.Release, 0, perPage)	for i := 1; ; i++ {		ls, _, err := g.client.Releases.ListReleases(g.repoID, &gitlab.ListReleasesOptions{			ListOptions: gitlab.ListOptions{				Page:    i,				PerPage: perPage,			},		}, nil, gitlab.WithContext(g.ctx))		if err != nil {			return nil, err		} 		for _, release := range ls {			releases = append(releases, g.convertGitlabRelease(release))		}		if len(ls) < perPage {			break		}	}	return releases, nil} type gitlabIssueContext struct {	IsMergeRequest bool} // GetIssues returns issues according start and limit////	Note: issue label description and colors are not supported by the go-gitlab library at this timefunc (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {	state := "all"	sort := "asc" 	if perPage > g.maxPerPage {		perPage = g.maxPerPage	} 	opt := &gitlab.ListProjectIssuesOptions{		State: &state,		Sort:  &sort,		ListOptions: gitlab.ListOptions{			PerPage: perPage,			Page:    page,		},	} 	allIssues := make([]*base.Issue, 0, perPage) 	issues, _, err := g.client.Issues.ListProjectIssues(g.repoID, opt, nil, gitlab.WithContext(g.ctx))	if err != nil {		return nil, false, fmt.Errorf("error while listing issues: %w", err)	}	for _, issue := range issues {		labels := make([]*base.Label, 0, len(issue.Labels))		for _, l := range issue.Labels {			labels = append(labels, &base.Label{				Name: strings.Replace(l, "::", "/", 1),			})		} 		var milestone string		if issue.Milestone != nil {			milestone = issue.Milestone.Title		} 		var reactions []*gitlab.AwardEmoji		awardPage := 1		for {			awards, _, err := g.client.AwardEmoji.ListIssueAwardEmoji(g.repoID, issue.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx))			if err != nil {				return nil, false, fmt.Errorf("error while listing issue awards: %w", err)			} 			reactions = append(reactions, awards...) 			if len(awards) < perPage {				break			} 			awardPage++		} 		allIssues = append(allIssues, &base.Issue{			Title:        issue.Title,			Number:       int64(issue.IID),			PosterID:     int64(issue.Author.ID),			PosterName:   issue.Author.Username,			Content:      issue.Description,			Milestone:    milestone,			State:        issue.State,			Created:      *issue.CreatedAt,			Labels:       labels,			Reactions:    g.awardsToReactions(reactions),			Closed:       issue.ClosedAt,			IsLocked:     issue.DiscussionLocked,			Updated:      *issue.UpdatedAt,			ForeignIndex: int64(issue.IID),			Context:      gitlabIssueContext{IsMergeRequest: false},		}) 		// record the issue IID, to be used in GetPullRequests()		g.iidResolver.recordIssueIID(issue.IID)	} 	return allIssues, len(issues) < perPage, nil} // GetComments returns comments according issueNumber// TODO: figure out how to transfer comment reactionsfunc (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) {	context, ok := commentable.GetContext().(gitlabIssueContext)	if !ok {		return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext())	} 	allComments := make([]*base.Comment, 0, g.maxPerPage) 	page := 1 	for {		var comments []*gitlab.Discussion		var resp *gitlab.Response		var err error		if !context.IsMergeRequest {			comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListIssueDiscussionsOptions{				Page:    page,				PerPage: g.maxPerPage,			}, nil, gitlab.WithContext(g.ctx))

Lines 1–500 of 794. Load more