JGit操作之2.0版
蒙才
背景
之前写了Java操作JGit的代码，用于记录和监督小组日常每天提交代码的次数和提交的数据量；用了一段时间，不是特别满意，又开发了一个2.0的版本，用于解决之前的痛点；
当前版本的具体功能如下：
- 通过传递参数，可以选择监控的仓库；
- 之前只是监控单分支，目前可以监控所有分支；
- 之前只是监控单分支所提交代码的情况，目前改成了监控本月月初到今天的代码提交情况；
我们开发的代码包括python和java两部分，这里只是提供java代码的核心部分；
代码信息
package com.xxx1.jgit;
import com.xxx1.util.JDBCUtils;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.api.LogCommand;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.RefAlreadyExistsException;
import org.eclipse.jgit.diff.*;
import org.eclipse.jgit.internal.storage.file.FileRepository;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.patch.FileHeader;
import org.eclipse.jgit.patch.HunkHeader;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.treewalk.AbstractTreeIterator;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import java.io.*;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.util.*;
/**
* @author xxx1
* @version 2.0
* @date 2022/1/11 18:09
* @Description: 进行git多分支处理
* 1.需要专门创建一个仓库，直接去clone，专门进行pull和checkout进行处理
* 2.进行pull
* 3.进行数据修改统计，并写入数据库中
* 4.进行checkout
**/
public class GitDemo3 {
public static CredentialsProvider usernamePasswordCredentialsProvider = new UsernamePasswordCredentialsProvider("username","password");
public static Map<String, String> branchMap = new HashMap<>();
public static void main(String[] args) throws GitAPIException, IOException, SQLException {
//1. 读取参数，创建git repository
String dir = args[0];
String projectName = dir.substring(dir.lastIndexOf('\\') + 1);
Git git = null;
Repository repository = null;
try{
repository = new FileRepositoryBuilder().setGitDir(Paths.get(dir,".git").toFile()).build();
git = new Git(repository);
}catch (IOException e){
e.printStackTrace();
}
// 2.执行git pull，拉取远程代码
git.pull().setCredentialsProvider(usernamePasswordCredentialsProvider).call();
// 3 .获取当前分支最近15天的提交信息
ArrayList<CommitLogInfo> results = new ArrayList<>();
getBranchCommitInfo(results, git, repository, projectName);
String currentBranchName = git.getRepository().getBranch();
// 4.切换至其他分支，并获取其信息(针对不同的repo拉取，可以配置化)
if("project1".equals(projectName)) {
// pycode做特殊处理
String url = "project1_url.git";
// 获取分支
Collection<Ref> refList = Git.lsRemoteRepository().setRemote(url).setCredentialsProvider(usernamePasswordCredentialsProvider).call();
// Collection<Ref> refList = repository.getAllRefs();
List<String> branchNameList = new ArrayList<>();
for (Ref ref : refList) {
String refName = ref.getName();
if (refName.startsWith("refs/heads/")) { //需要进行筛选
String branchName = refName.replace("refs/heads/", "");
branchNameList.add(branchName);
}
}
// 对每个分支进行数据统计
for (String branchName : branchNameList) {
if (branchName.equals(currentBranchName) || branchName.equals("master") || branchName.equals("dev") || branchName.equals("backup") || branchName.equals("null")) {
continue;
} else {
try{
git.checkout().setCreateBranch(true).setName(branchName).setStartPoint("origin/" + branchName).call();
}catch(RefAlreadyExistsException e){
System.out.println("branch:" + branchName + " already Exists");
// git.branchCreate().setName(branchName).setStartPoint("origin/" + branchName).call();
git.checkout().setCreateBranch(false).setName(branchName).setStartPoint("origin/" + branchName).call();
}
git.pull().setCredentialsProvider(usernamePasswordCredentialsProvider).call();//拉取最新的提交 }
getBranchCommitInfo(results, git, repository, projectName);
}
}
}else if("project2".equals(projectName)){
// LV95.02 MySQL 只有master分区
String url = "project2_url.git";
Collection<Ref> refList = Git.lsRemoteRepository().setRemote(url).setCredentialsProvider(usernamePasswordCredentialsProvider).call();
getOtherBranchCommitInfo(usernamePasswordCredentialsProvider,refList,currentBranchName,git,repository,projectName,results);
}else if("project3".equals(projectName)){
String url = "project3_url.git";
Collection<Ref> refList = Git.lsRemoteRepository().setRemote(url).setCredentialsProvider(usernamePasswordCredentialsProvider).call();
getOtherBranchCommitInfo(usernamePasswordCredentialsProvider,refList,currentBranchName,git,repository,projectName,results);
}else if("project4".equals(projectName)){
String url = "project4_url.git";
Collection<Ref> refList = Git.lsRemoteRepository().setRemote(url).setCredentialsProvider(usernamePasswordCredentialsProvider).call();
getOtherBranchCommitInfo(usernamePasswordCredentialsProvider,refList,currentBranchName,git,repository,projectName,results);
}else if("project5".equals(projectName)){
String url = "project5_url.git";
Collection<Ref> refList = Git.lsRemoteRepository().setRemote(url).setCredentialsProvider(usernamePasswordCredentialsProvider).call();
getOtherBranchCommitInfo(usernamePasswordCredentialsProvider,refList,currentBranchName,git,repository,projectName,results);
}else{
}
// 切换至原始的分支
git.checkout().setCreateBranch(false).setStartPoint("origin/" + currentBranchName).setName(currentBranchName).call();
// 5.将数据写入到MySQL中
// 4.mysql写入数据库中
InputStream inputStream = new BufferedInputStream(new FileInputStream("src/resources/config.properties"));
Properties properties = new Properties();
properties.load(inputStream);
String username = properties.getProperty("username");
String password = properties.getProperty("password");
String driverClass = properties.getProperty("driverClass");
String url = properties.getProperty("url");
Connection connection = JDBCUtils.createConnection(driverClass, url, username, password);
String stmtStr = "insert into schema.git_commit_info2(data_dt,project_name,branch_name,commit_id,commit_name,commit_email,commit_comment,add_lines,sub_lines,commit_time) values";
long num = 0;
int bulkNum = 20;
Statement statement = connection.createStatement();
for(CommitLogInfo commitLogInfo : results){
String sql_val = String.format("(%s,\'%s\',\'%s\',\'%s\',\'%s\',\'%s\',\'%s\',\'%s\',\'%s\',\'%s\'),","current_date",commitLogInfo.getProjectName(),commitLogInfo.getBranchName(),commitLogInfo.getCommitID(), commitLogInfo.getCommitUserName(),commitLogInfo.getCommitEmail(),commitLogInfo.getCommitComment(),commitLogInfo.getAddLines(),commitLogInfo.getSubLines(),commitLogInfo.getCommitTime());
stmtStr = stmtStr + sql_val;
num = num + 1;
if(num % bulkNum == 0 || num == results.size()){
stmtStr = stmtStr.substring(0,stmtStr.lastIndexOf(","));
stmtStr = stmtStr + " ON DUPLICATE KEY UPDATE update_time=current_timestamp";
statement.addBatch(stmtStr);
if(num % bulkNum == 0 || num == results.size()){
System.out.println(stmtStr);
statement.executeBatch();
statement.clearBatch();
}
stmtStr ="insert into schema.git_commit_info2(data_dt,project_name,branch_name,commit_id,commit_name,commit_email,commit_comment,add_lines,sub_lines,commit_time) values";
}
}
}
public static boolean branNameExists(Git git, String branchName) throws GitAPIException {
List<Ref> refs = git.branchList().call();
for(Ref ref : refs){
if(ref.getName().contains(branchName)){
return true;
}
}
return false;
}
public static void getOtherBranchCommitInfo(CredentialsProvider usernamePasswordCredentialsProvider,Collection<Ref> refList, String currentBranchName, Git git, Repository repository, String projectName,ArrayList<CommitLogInfo> results) throws GitAPIException, IOException {
List<String> branchNameList = new ArrayList<>();
for (Ref ref : refList) {
String refName = ref.getName();
if (refName.startsWith("refs/heads/")) { //需要进行筛选
String branchName = refName.replace("refs/heads/", "");
branchNameList.add(branchName);
}
}
// 对每个分支进行数据统计
for (String branchName : branchNameList) {
if (branchName.equals(currentBranchName)) {
continue;
} else {
try {
git.checkout().setCreateBranch(true).setName(branchName).setStartPoint("origin/" + branchName).call();
//git.checkout().setCreateBranch(true).setName(branchName).call();
} catch(RefAlreadyExistsException e){
System.out.println("branch:" + branchName + " already exists");
git.checkout().setCreateBranch(false).setName(branchName).setStartPoint("origin/" + branchName).call();
// git.branchCreate().setName(branchName).setStartPoint("origin/" + branchName).call();
}
git.pull().setCredentialsProvider(usernamePasswordCredentialsProvider).call();//拉取最新的提交 }
getBranchCommitInfo(results, git, repository, projectName);
}
}
}
public static void getBranchCommitInfo(ArrayList<CommitLogInfo> results,Git git, Repository repository, String projectName) throws GitAPIException, IOException {
LogCommand logCommand = git.log();
String branchName = git.getRepository().getBranch();
System.out.println(projectName + "-" + branchName);
Iterable<RevCommit> call = logCommand.call();
java.util.Date nowDate = new java.util.Date();
Calendar calendar = Calendar.getInstance();
calendar.setTime(nowDate);
calendar.add(Calendar.DAY_OF_MONTH, -15);
java.util.Date yesterday = calendar.getTime();
Timestamp yesterdayTimestamp = new Timestamp(yesterday.getTime());
calendar.setTime(nowDate);
calendar.add(Calendar.DAY_OF_MONTH, -366);
java.util.Date yearTime = calendar.getTime();
Timestamp yearTimestamp = new Timestamp(yearTime.getTime());
ArrayList<RevCommit> arrayList = new ArrayList<>();
for(RevCommit revCommit : call){
long commitTime = revCommit.getCommitTime() * 1000L;
Timestamp runningTimestamp = new Timestamp(commitTime);
//需要添加初始节点，解决第一次处理的问题
if(runningTimestamp.compareTo(yearTimestamp) < 0){
arrayList.add(revCommit);
break;
}else{
arrayList.add(revCommit);
}
}
int length = arrayList.size();
for(int i = 0; i <= length - 2; i++){
RevCommit revCommit = arrayList.get(i);
RevCommit prevCommit = arrayList.get(i + 1);
String commitID = revCommit.getId().toString().split(" ")[1];
String commitComment = revCommit.getFullMessage().replaceAll("'","").replaceAll("\n","\t");
PersonIdent personIdent = revCommit.getCommitterIdent();
String commitName = personIdent.getName();
String commitEmail = personIdent.getEmailAddress();
long commitTime = revCommit.getCommitTime() * 1000L;
Timestamp runningTimestamp = new Timestamp(commitTime);
// 只获取7天的数据
if(runningTimestamp.compareTo(yesterdayTimestamp) >= 0){
// 获取变化的行数
CommitLines commitLines = getCommitLines2(git,prevCommit,revCommit,repository);
CommitLogInfo commitLogInfo = new CommitLogInfo();
commitLogInfo.setProjectName(projectName);
commitLogInfo.setBranchName(branchName);
commitLogInfo.setCommitID(commitID);
commitLogInfo.setCommitUserName(commitName);
commitLogInfo.setCommitEmail(commitEmail);
commitLogInfo.setCommitComment(commitComment);
commitLogInfo.setCommitTime(runningTimestamp);
commitLogInfo.setAddLines(commitLines.getAddLines());
commitLogInfo.setSubLines(commitLines.getSubLines());
results.add(commitLogInfo);
}
}
}
/**
*
* @param git
* @param headCommit
* @param currentCommit
* @param repository
* @return
* @throws GitAPIException
* @throws IOException
*/
public static CommitLines getCommitLines2(Git git,RevCommit headCommit,RevCommit currentCommit, Repository repository) throws GitAPIException, IOException {
List<DiffEntry> diffs = null;
AbstractTreeIterator oldTreeIter = prepareTreeParser(repository, headCommit);
AbstractTreeIterator newTreeIter = prepareTreeParser(repository, currentCommit);
diffs = git.diff().setNewTree(newTreeIter).setOldTree(oldTreeIter).call();
ByteArrayOutputStream out = new ByteArrayOutputStream();
DiffFormatter df = new DiffFormatter(out);
df.setDiffComparator(RawTextComparator.WS_IGNORE_ALL);
df.setRepository(repository);
int addLines = 0;
int subLines = 0;
for(DiffEntry diffEntry: diffs){
df.format(diffEntry);
String diffText = out.toString("UTF-8");
FileHeader fileHeader = df.toFileHeader(diffEntry);
List<HunkHeader> hunks = (List<HunkHeader>)fileHeader.getHunks();
int addSize = 0;
int subSize = 0;
for(HunkHeader hunkHeader : hunks){
EditList editList = hunkHeader.toEditList();
for(Edit edit : editList){
subSize = subSize + edit.getEndA() - edit.getBeginA();
addSize = addSize + edit.getEndB() - edit.getBeginB();
}
}
out.reset();
addLines += addSize;
subLines += subSize;
}
return new CommitLines(addLines, subLines);
}
public static AbstractTreeIterator prepareTreeParser(Repository repository,RevCommit commit){
System.out.println(commit.getId());
try (RevWalk walk = new RevWalk(repository)) {
System.out.println(commit.getTree().getId());
RevTree tree = walk.parseTree(commit.getTree().getId());
CanonicalTreeParser oldTreeParser = new CanonicalTreeParser();
try (ObjectReader oldReader = repository.newObjectReader()) {
oldTreeParser.reset(oldReader, tree.getId());
}
walk.dispose();
return oldTreeParser;
}catch (Exception e) {
e.printStackTrace();
}
return null;
}
}
工作流程
1.接收传递的repo name
2.进行认证和登录，针对当前分支进行pull操作，并统计当前分支代码提交情况，放入指定的类里面
3.对其他分支进行处理，统计其他分支的代码提交情况
4.将统计的信息写入数据库中；
注：1.这里对project1没有进行函数封装，实际上和project2，project3等的处理逻辑基本一致；
2.我们对project1还是做了一下特殊的处理，由于project1的工程项目比较大，有些分支代码特别多，进行checkout特别慢，但是这些分支的代码基本不提交，因此对这些分支不进行统计；
3.我们对branch的情况，都是先要在远程仓库中拉取的，防止远程创建了新的branch而本地不知道，导致对新的branch没有及时的统计，不需要人工的干预；
4.我们对分支进行checkout时也做了处理，先假设本地有branch进行checkout，如果失败了，则采用其他的方式进行checkout；
总结
以下是对整体工程的总结，包括java和Python代码；
需求：
利用JGit对本组管理的所有项目和工程进行代码提交统计，主要是统计当月代码提交的代码量和代码提交次数(代码提交次数一天只能算一次，即使一天提交了多次)；
解决思路
在windows本机上设置定时任务，调用bat程序，bat程序调用python程序，python程序调用java jar包的程序，以下是各个程序的功能：
windows定时任务：设置每周三和每周五跑数，调用bat程序；
bat程序：只是方便windows定时任务调用，执行代码为python xxx.py；
python程序:分别传递不同的repo name，执行java的jar包，将不同repo最近的代码提交信息写入到数据库中，并发送短信给相应的负责人；
java程序：接收传递参数repo name，统计该repo下不同branch最近15天的代码提交情况，并写入到数据库中；
类似资料：