⁵

自定义GitHub仓库文档读取器

青春的疯子

2025-05-12 22:47

阅读675

青春的疯子

2025-05-12 22:47

阅读 675

一向不喜欢废话，摸索两小时，干货奉上，希望大家点点赞

一、引入依赖

▼
xml
复制代码
<!-- GitHub API -->
<dependency>
  <groupId>org.kohsuke</groupId>
  <artifactId>github-api</artifactId>
  <version>1.318</version>
</dependency>

二、具体实现

▼
java
复制代码
/**
 * @version v1.0.0
 * @belongsProject: feng-ai-agent
 * @belongsPackage: com.qcdfz.fengaiagent.rag
 * @author: fgh
 * @description: GitHub文档加载器 用于加载GitHub仓库中的文档内容
 * @createTime: 2025-05-12 16:53
 */
@Slf4j
public class GitHubDocumentLoader {

    private final GitHub gitHub;
    private final String owner;
    private final String repo;
    private final String branch;

    /**
     * 构造函数
     *
     * @param gitHub GitHub客户端实例
     * @param owner 仓库所有者
     * @param repo 仓库名称
     * @param branch 分支名称
     */
    public GitHubDocumentLoader(GitHub gitHub, String owner, String repo, String branch) {
        Assert.notNull(gitHub, "GitHub实例不能为空");
        Assert.notNull(owner, "仓库所有者不能为空");
        Assert.notNull(repo, "仓库名称不能为空");
        this.gitHub = gitHub;
        this.owner = owner;
        this.repo = repo;
        this.branch = branch != null ? branch : "main";
    }

    /**
     * 加载单个文件内容
     *
     * @param path 文件路径
     * @return Document对象
     */
    public Document loadDocument(String path) {
        try {
            GHContent content = getRepository().getFileContent(path, branch);
            Assert.isTrue(content.isFile(), "路径必须指向文件");
            return createDocument(content);
        } catch (IOException e) {
            log.error("从GitHub加载文档失败: {}", path, e);
            throw new RuntimeException("从GitHub加载文档失败: " + path, e);
        }
    }

    /**
     * 加载目录下的所有文件
     *
     * @param path 目录路径
     * @return Document列表
     */
    public List<Document> loadDocuments(String path) {
        List<Document> documents = new ArrayList<>();
        try {
            List<GHContent> contents = getRepository().getDirectoryContent(path, branch);
            for (GHContent content : contents) {
                if (content.isFile()) {
                    documents.add(createDocument(content));
                } else if (content.isDirectory()) {
                    documents.addAll(loadDocuments(content.getPath()));
                }
            }
        } catch (IOException e) {
            log.error("从GitHub加载文档失败: {}", path, e);
            throw new RuntimeException("从GitHub加载文档失败: " + path, e);
        }
        return documents;
    }

    /**
     * 获取仓库信息
     *
     * @return 仓库信息Map
     */
    public Map<String, Object> getRepositoryInfo() {
        try {
            GHRepository repository = getRepository();
            return Map.of(
                "name", Objects.toString(repository.getName(), ""),
                "description", Objects.toString(repository.getDescription(), ""),
                "stars", repository.getStargazersCount(),
                "forks", repository.getForksCount(),
                "language", Objects.toString(repository.getLanguage(), ""),
                "defaultBranch", Objects.toString(repository.getDefaultBranch(), ""),
                "htmlUrl", repository.getHtmlUrl() != null ? repository.getHtmlUrl().toString() : "",
                "cloneUrl", Objects.toString(repository.getHttpTransportUrl(), "")
            );
        } catch (IOException e) {
            log.error("获取仓库信息失败: {}/{}", owner, repo, e);
            throw new RuntimeException("获取仓库信息失败: " + owner + "/" + repo, e);
        }
    }

    /**
     * 获取仓库对象
     *
     * @return GHRepository对象
     * @throws IOException 如果获取失败
     */
    private GHRepository getRepository() throws IOException {
        return gitHub.getRepository(owner + "/" + repo);
    }

    /**
     * 创建Document对象
     *
     * @param content GitHub内容对象
     * @return Document对象
     * @throws IOException 如果读取失败
     */
    private Document createDocument(GHContent content) throws IOException {
        String text = content.getContent();
        Map<String, Object> metadata = Map.of(
            "github_git_url", content.getGitUrl(),
            "github_download_url", content.getDownloadUrl(),
            "github_html_url", content.getHtmlUrl(),
            "github_url", content.getUrl(),
            "github_file_name", content.getName(),
            "github_file_path", content.getPath(),
            "github_file_sha", content.getSha(),
            "github_file_size", Long.toString(content.getSize()),
            "github_file_encoding", content.getEncoding()
        );
        return new Document(text, metadata);
    }


    /**
     * 创建Builder实例
     *
     * @return Builder实例
     */
    public static Builder builder() {
        return new Builder();
    }

    /**
     * 构建器类
     */
    public static class Builder {
        private GitHub gitHub;
        private String owner;
        private String repo;
        private String branch;

        /**
         * 设置GitHub客户端实例
         *
         * @param gitHub GitHub客户端实例
         * @return Builder实例
         */
        public Builder gitHub(GitHub gitHub) {
            this.gitHub = gitHub;
            return this;
        }

        /**
         * 设置仓库所有者
         *
         * @param owner 仓库所有者
         * @return Builder实例
         */
        public Builder owner(String owner) {
            this.owner = owner;
            return this;
        }

        /**
         * 设置仓库名称
         *
         * @param repo 仓库名称
         * @return Builder实例
         */
        public Builder repo(String repo) {
            this.repo = repo;
            return this;
        }

        /**
         * 设置分支名称
         *
         * @param branch 分支名称
         * @return Builder实例
         */
        public Builder branch(String branch) {
            this.branch = branch;
            return this;
        }

        /**
         * 构建GitHubDocumentLoader实例
         *
         * @return GitHubDocumentLoader实例
         */
        public GitHubDocumentLoader build() {
            return new GitHubDocumentLoader(gitHub, owner, repo, branch);
        }
    }
}

这个是仿照SpringAi官方的DocumentReader的Builder构造器实现的，但是我并没有去实现他的DocumentReader接口，这里只是一个例子，需要其他增强，自己实现即可

三、使用方法

▼
java
复制代码
@SpringBootTest
class GitHubDocumentLoaderTest {

    @Test
    public void test() throws IOException {
        String token = GlobalApiKey.GITHUB_TOKEN;

        // 推荐用GitHubBuilder
        GitHub github = new GitHubBuilder().withOAuthToken(token).build();
        GitHubDocumentLoader loader = GitHubDocumentLoader.builder()
                .gitHub(github)
                .owner("fenghuanghuai")
                .repo("hell-world")
                .branch("main")
                .build();
        // 读取文件 下面有实现效果
        Document doc = loader.loadDocument("/test/test.md");
        // 读取目录 下面有实现效果
        List<Document> docs = loader.loadDocuments("/");
        // 获取仓库信息 下面有实现效果
        Map<String, Object> repoInfo = loader.getRepositoryInfo();
        Assertions.assertNotNull(repoInfo);
    }
}

这个是我的仓库内容