实现简单的自动部署
Intro
前段时间在帮忙改一个纯前端的 vue 项目,部署在 IIS 上的,之前每次都要手动去部署,于是想实现一个自动部署,项目是 Github 上的一个项目,Github 上没有比较好用的 IIS 部署的 Action,于是想自己写一个自动部署的程序实现自动部署,基于 Github 的 WebHook 来收到更新再执行部署,之前简单写过 Github WebHook 的处理,可以参考:GetStarted with Github Webhook
How
大概的实现是基于 Channel
实现一个 InMemory 的 queue,在收到 Github 的 WebHook Push 之后,向 queue 里写入一条消息,然后返回 response
另外添加一个 BackgroundService
来消费 queue 里的消息,执行我们的实际部署
部署前的准备:
准备一个代码库用来更新代码
安装 git/nodejs/
yarn
等
Implement
来看一些实现细节:
WebHookEventProcessor
如下:
public sealed class MyWebhookEventProcessor: WebhookEventProcessor
{
private readonly IEventPublisher _eventPublisher;
private readonly ILogger<MyWebhookEventProcessor> _logger;
public MyWebhookEventProcessor(IEventPublisher eventPublisher, ILogger<MyWebhookEventProcessor> logger)
{
_eventPublisher = eventPublisher;
_logger = logger;
}
protected override async Task ProcessPushWebhookAsync(WebhookHeaders headers, PushEvent pushEvent)
{
var (repoName, repoFullName, commitId, commitMsg) = (pushEvent.Repository?.Name, pushEvent.Repository?.FullName, pushEvent.HeadCommit?.Id, pushEvent.HeadCommit?.Message);
var (name, email) = (pushEvent.Pusher.Name, pushEvent.Pusher.Email);
if (string.IsNullOrEmpty(commitId)
|| string.IsNullOrEmpty(commitMsg)
|| string.IsNullOrEmpty(repoName)
|| commitMsg.IndexOf("skip-ci", StringComparison.OrdinalIgnoreCase) > -1
|| commitMsg.IndexOf("skip-cd", StringComparison.OrdinalIgnoreCase) > -1
|| commitMsg.IndexOf("cd-skip", StringComparison.OrdinalIgnoreCase) > -1
)
{
return;
}
_logger.LogInformation("Push event received {RepoName} {CommitId} {CommitMsg} {PushByName} {PushByEmail}",
repoName, commitId, commitMsg, name, email);
// process push event
var githubPushEvent = new GithubPushEvent
{
RepoName = repoName,
RepoFullName = repoFullName ?? repoName,
CommitId = commitId,
CommitMsg = commitMsg,
Timestamp = DateTimeOffset.Parse(pushEvent.HeadCommit!.Timestamp),
PushByName = name,
PushByEmail = email ?? string.Empty
};
await _eventPublisher.PublishAsync(githubPushEvent);
}
}
我们可以从 push event 的信息里找到 commit 相关的信息,这里加了一个判断如果 commit message 里包含了 skip-ci
/skip-cd
/cd-skip
类的信息,我们会将不会进行部署,之后是通过一个 EventPublisher
将 push event 发布到 queue 里
EventPublisher
和 EventHandler
放在了一起,实现代码如下:
public sealed class EventHandler : BackgroundService, IEventPublisher
{
private readonly ILogger<EventHandler> _logger;
private readonly IConfiguration _configuration;
private readonly IDeployHistoryRepository _deployHistoryRepository;
private readonly Channel<GithubPushEvent> _channel =
Channel.CreateBounded<GithubPushEvent>(new BoundedChannelOptions(3)
{
FullMode = BoundedChannelFullMode.DropOldest
});
public EventHandler(ILogger<EventHandler> logger, IConfiguration configuration, IDeployHistoryRepository deployHistoryRepository)
{
_logger = logger;
_configuration = configuration;
_deployHistoryRepository = deployHistoryRepository;
}
private readonly TimeSpan _period = TimeSpan.FromSeconds(10);
public bool Publish<TEvent>(TEvent @event) where TEvent : class, IEventBase
{
throw new NotImplementedException();
}
public async Task<bool> PublishAsync<TEvent>(TEvent @event) where TEvent : class, IEventBase
{
if (@event is not GithubPushEvent githubPushEvent)
{
throw new NotSupportedException();
}
await _channel.Writer.WriteAsync(githubPushEvent);
return true;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await foreach (var githubPushEvent in _channel.Reader.ReadAllAsync(stoppingToken))
{
var beginTime = DateTimeOffset.Now;
try
{
await HandleGithubPushEvent(githubPushEvent);
var endTime = DateTimeOffset.Now;
var elapsed = endTime - beginTime;
_logger.LogInformation("{RepoName} Deploy done in {Elapsed}, last commit msg: {CommitMsg}, {PushedBy}, please help check the result",
githubPushEvent.RepoName, elapsed, githubPushEvent.CommitMsg, githubPushEvent.PushByEmail);
var deployHistory = new DeployHistory
{
Event = githubPushEvent,
BeginTime = beginTime,
EndTime = endTime,
Elapsed = elapsed
};
_deployHistoryRepository.AddDeployHistory(githubPushEvent.RepoName, deployHistory);
}
catch (Exception e)
{
_logger.LogError(e, "{Method} Exception", nameof(HandleGithubPushEvent));
}
await Task.Delay(_period, stoppingToken);
}
}
private async Task HandleGithubPushEvent(GithubPushEvent githubPushEvent)
{
// find repo, exec git pull
var repoRoot = _configuration.GetRequiredAppSetting("RepoRoot");
var repoFolder = Path.Combine(repoRoot, githubPushEvent.RepoName);
if (!Directory.Exists(repoFolder))
{
throw new InvalidOperationException($"Repo({githubPushEvent.RepoName}) not exists in path {repoFolder}");
}
if (_configuration.GetAppSetting("PreferLibGit2Sharp", false))
{
using var repo = new Repository(repoFolder);
// Credential information to fetch
var options = new PullOptions
{
FetchOptions = new FetchOptions
{
CredentialsProvider = (_, _, _) =>
new UsernamePasswordCredentials()
{
Username = _configuration["GitCredential:Name"],
Password = _configuration["GitCredential:Token"]
}
} };
// User information to create a merge commit
var signature = new Signature(new Identity(_configuration["GitCredential:Name"], _configuration["GitCredential:Email"]), DateTimeOffset.Now);
// Pull
RetryHelper.TryInvoke(() => Commands.Pull(repo, signature, options), 10);
}
else
{
var gitPath = ApplicationHelper.ResolvePath("git") ?? _configuration.GetRequiredAppSetting("GitPath");
var gitPullResult = await RetryHelper.TryInvokeAsync(() => CommandExecutor.ExecuteAndCaptureAsync(gitPath, "pull", repoFolder)!,
r => r?.ExitCode == 0, 10);
if (gitPullResult?.ExitCode != 0)
{
throw new InvalidOperationException($"Error when git pull, exitCode: {gitPullResult?.ExitCode}, {gitPullResult?.StandardError}");
}
}
var nodePath = _configuration.GetRequiredAppSetting("NodePath");
var previousPath = Environment.GetEnvironmentVariable("PATH") ?? string.Empty;
var updatedPath = previousPath.EndsWith(';') ? $"{previousPath}{nodePath}" : $"{previousPath};{nodePath}";
_logger.LogInformation("Previous environment path: {PreviousPath}, updatedPath: {UpdatedPath}", previousPath, updatedPath);
var yarnPath = ApplicationHelper.ResolvePath("yarn.cmd") ?? _configuration.GetRequiredAppSetting("YarnPath");
// exec yarn
var yarnResult = await CommandExecutor.ExecuteAndCaptureAsync(yarnPath, null, repoFolder, info =>
{
if (_configuration.GetAppSetting<bool>("AddNodeOptionsEnv"))
info.EnvironmentVariables.Add("NODE_OPTIONS", "--openssl-legacy-provider");
info.EnvironmentVariables["NODE_PATH"] = nodePath;
info.EnvironmentVariables["PATH"] = updatedPath;
var processUser = _configuration["ProcessUserCredential:UserName"];
if (!string.IsNullOrEmpty(processUser))
{
info.UserName = processUser;
if (OperatingSystem.IsWindows() && !string.IsNullOrEmpty(_configuration["ProcessUserCredential:Password"]))
info.PasswordInClearText = Convert.FromBase64String(_configuration["ProcessUserCredential:Password"]).GetString();
}
});
if (yarnResult.ExitCode != 0)
{
_logger.LogError("Error when yarn, exitCode: {ExitCode}, output: {Output}, error: {Error}",
yarnResult.ExitCode, yarnResult.StandardOut, yarnResult.StandardError);
throw new InvalidOperationException($"Error when yarn, exitCode: {yarnResult.ExitCode}");
}
// cleanup previous dist folder
var distFolder = Path.Combine(repoFolder, "dist");
if (Directory.Exists(distFolder))
Directory.Delete(distFolder, true);
// exec yarn build
var buildResult = await CommandExecutor.ExecuteAndCaptureAsync(yarnPath, "build", repoFolder, info =>
{
if (_configuration.GetAppSetting<bool>("AddNodeOptionsEnv"))
info.EnvironmentVariables.Add("NODE_OPTIONS", "--openssl-legacy-provider");
info.EnvironmentVariables["NODE_PATH"] = nodePath;
info.EnvironmentVariables["PATH"] = updatedPath;
var processUser = _configuration["ProcessUserCredential:UserName"];
if (!string.IsNullOrEmpty(processUser))
{
info.UserName = processUser;
if (OperatingSystem.IsWindows() && !string.IsNullOrEmpty(_configuration["ProcessUserCredential:Password"]))
info.PasswordInClearText = Convert.FromBase64String(_configuration["ProcessUserCredential:Password"]).GetString();
}
});
if (buildResult.ExitCode != 0)
{
_logger.LogError("Error when yarn build, exitCode: {ExitCode}, output: {Output}, error: {Error}",
buildResult.ExitCode, buildResult.StandardOut, buildResult.StandardError);
throw new InvalidOperationException($"Error when yarn build, exitCode: {buildResult.ExitCode}");
}
// copy dist to site folder
var siteFolder = _configuration[$"AppSettings:RepoSiteMappings:{githubPushEvent.RepoName}"];
if (string.IsNullOrEmpty(siteFolder))
{
_logger.LogError("No site name mapped, RepoName: {RepoName}", githubPushEvent.RepoName);
throw new InvalidOperationException($"Error when yarn build, exitCode: {buildResult.ExitCode}");
}
CopyDirectory(distFolder, siteFolder, true);
}
// https://learn.microsoft.com/en-us/dotnet/standard/io/how-to-copy-directories
private static void CopyDirectory(string sourceDir, string destinationDir, bool recursive)
{
// Get information about the source directory
var dir = new DirectoryInfo(sourceDir);
// Check if the source directory exists
if (!dir.Exists)
throw new DirectoryNotFoundException($"Source directory not found: {dir.FullName}");
// Cache directories before we start copying
var dirs = dir.GetDirectories();
// Create the destination directory
Directory.CreateDirectory(destinationDir);
// Get the files in the source directory and copy to the destination directory
foreach (var file in dir.GetFiles())
{
var targetFilePath = Path.Combine(destinationDir, file.Name);
file.CopyTo(targetFilePath, true);
}
// If recursive and copying subdirectories, recursively call this method
if (recursive)
{
foreach (var subDir in dirs)
{
var newDestinationDir = Path.Combine(destinationDir, subDir.Name);
CopyDirectory(subDir.FullName, newDestinationDir, true);
}
}
}
}
这里使用 Channel 来实现一个 InMemory 的 queue 并且指定了 queue 里最多存 3 条 数据,如果超出了就删掉最老的消息
Channel.CreateBounded<GithubPushEvent>(new BoundedChannelOptions(3)
{
FullMode = BoundedChannelFullMode.DropOldest
})
前面的 PublishAsync
就是一个简单的写入 event 到我们 Channel
也是我们的 InMemory quque await _channel.Writer.WriteAsync(githubPushEvent);
消费消息通过直接读 channel 的消息
await foreach (var githubPushEvent in _channel.Reader.ReadAllAsync(stoppingToken))
{
// ... handle github push event
}
接着来看下部署的过程吧,也就是 HandleGithubPushEvent
方法的逻辑
第一步我们会先检查一下代码库对应的目录是否存在,如果不存在则报错,不能继续后续的操作
第二步就是在我们的代码库下进行
git pull
来获取最新的代码,这里写了两种实现一种是直接开一个进程执行命令git pull
,另外一种实现是使用LibGit2Sharp
来实现第三步是执行
yarn
来还原依赖第四步是删除原来的 dist 目录如果存在的话
第五步是执行
yarn build
来生成要部署的 dist 文件最后一步是将新生成的 dist 文件拷贝到网站对应的目录下
看过最初版代码的话会发现现在的代码实际会更加复杂一些,因为在部署的时候遇到了各种各样的问题,最后发现实际执行的用户并不是默认的用户,一些环境变量是没有的,导致执行命令一直有问题,后面加了
git
和yarn
的路径配置,找不到的话就用 config 里的配置,并且新增了LibGit2Sharp
的git
实现
部署成功之后会记录一个 Deploy History,这样我们可以知道有没有部署完成,这里其实想接一个 webhook 进行推送通知,但是没有好的推送通知的地方,就做了一个简化,记录一下 deploy history 并且提供 API 去查询 deploy history
deploy history 的存储非常的简单,基于 InMemory 的 ConcurrentQueue
实现最多保存最新的十条记录
public interface IDeployHistoryRepository
{
void AddDeployHistory(string service, DeployHistory deployHistory);
DeployHistory[] GetDeployHistory(string service);
IReadOnlyDictionary<string, DeployHistory[]> GetAllDeployHistory();
}
public class DeployHistoryRepository: IDeployHistoryRepository
{
private readonly ConcurrentDictionary<string, ConcurrentQueue<DeployHistory>> _store = new();
private const int MaxDeployHistoryCount = 10;
public void AddDeployHistory(string service, DeployHistory deployHistory)
{
var svcStore = _store.GetOrAdd(service, _ => new());
svcStore.Enqueue(deployHistory);
if (svcStore.Count > MaxDeployHistoryCount)
{
svcStore.TryDequeue(out _);
}
}
public DeployHistory[] GetDeployHistory(string service)
{
if (_store.TryGetValue(service, out var svcStore))
return svcStore.OrderByDescending(x => x.BeginTime).ToArray();
return Array.Empty<DeployHistory>();
}
public IReadOnlyDictionary<string, DeployHistory[]> GetAllDeployHistory()
{
return _store.ToDictionary(
x => x.Key,
x => x.Value.OrderByDescending(h => h.BeginTime).ToArray()
);
}
}
查询的 API 实现如下,就是直接从 DeployHistoryRepository
里查数据
app.Map("/deploy-history", (IDeployHistoryRepository repository) => repository.GetAllDeployHistory());
app.Map("/deploy-history/{service}", (string service, IDeployHistoryRepository repository) => repository.GetDeployHistory(service));
Test
为了方便测试,增加了一个测试的 API,就是直接向 queue 里添加一个消息
app.MapPost("/deploy-test", async (IEventPublisher eventPublisher) =>
{
var githubPushEvent = new GithubPushEvent
{
RepoName = "NetConfChina_Frontend",
RepoFullName = "NetConfChina_Frontend",
CommitId = "x",
CommitMsg = "test",
Timestamp = DateTimeOffset.Now,
PushByName = "Test",
PushByEmail = "weihanli@outlook.com"
};
await eventPublisher.PublishAsync(githubPushEvent);
});
我们来测试一下,首先来获取一下 deploy history 会返回一个空对象,之后我们触发一下 deploy
,之后重新请求 deploy history API
可以看到已经部署完成了,我们可以去网站目录下看看有没有生成文件,可以看到,文件已经生成并复制到了网站目录下了,对比文件的修改时间可以知道是刚生成的文件
More
IIS 默认 20 分钟没有请求会休眠,如果 hook handler 在 IIS 上 in-process 来部署,可能会出现 deploy history 过一段时间之后就没有了,这个是 IIS 的配置问题,如果不想改 IIS 的配置,可以写一个定时任务每分钟跑一次 hook handler 的接口,这样来做一个健康检查不仅可以监控 hook handler 的健康状态也可以实现 IIS process 的保活,就不会出现休眠的情况了
References
https://github.com/WeihanLi/SamplesInPractice/tree/main/GitHookSample