| | | 1 | | |
| | | 2 | | using System.Threading.Channels; |
| | | 3 | | |
| | | 4 | | namespace Kestrun.Callback; |
| | | 5 | | |
| | | 6 | | /// <summary> |
| | | 7 | | /// Background worker that processes callback requests from the queue. |
| | | 8 | | /// </summary> |
| | | 9 | | /// <remarks> |
| | | 10 | | /// Initializes a new instance of the <see cref="CallbackWorker"/> class. |
| | | 11 | | /// </remarks> |
| | | 12 | | /// <param name="queue"> The in-memory callback queue.</param> |
| | | 13 | | /// <param name="sender"> The callback sender.</param> |
| | | 14 | | /// <param name="retry"> The callback retry policy.</param> |
| | | 15 | | /// <param name="log"> The logger.</param> |
| | | 16 | | /// <param name="store"> The optional callback store.</param> |
| | 2 | 17 | | public sealed class CallbackWorker( |
| | 2 | 18 | | InMemoryCallbackQueue queue, |
| | 2 | 19 | | ICallbackSender sender, |
| | 2 | 20 | | ICallbackRetryPolicy retry, |
| | 2 | 21 | | Serilog.ILogger log, |
| | 2 | 22 | | ICallbackStore? store = null) : BackgroundService |
| | | 23 | | { |
| | 2 | 24 | | private readonly ChannelReader<CallbackRequest> _reader = queue.Channel.Reader; |
| | 2 | 25 | | private readonly ChannelWriter<CallbackRequest> _writer = queue.Channel.Writer; |
| | 2 | 26 | | private readonly ICallbackSender _sender = sender; |
| | 2 | 27 | | private readonly ICallbackRetryPolicy _retry = retry; |
| | 2 | 28 | | private readonly ICallbackStore? _store = store; // optional |
| | 2 | 29 | | private readonly Serilog.ILogger _log = log; |
| | | 30 | | |
| | | 31 | | /// <summary> |
| | | 32 | | /// Executes the background service. |
| | | 33 | | /// </summary> |
| | | 34 | | /// <param name="stoppingToken"> Cancellation token to stop the service.</param> |
| | | 35 | | /// <returns> A task that represents the background service execution.</returns> |
| | | 36 | | protected override async Task ExecuteAsync(CancellationToken stoppingToken) |
| | | 37 | | { |
| | | 38 | | try |
| | | 39 | | { |
| | 10 | 40 | | await foreach (var req in _reader.ReadAllAsync(stoppingToken)) |
| | | 41 | | { |
| | | 42 | | // Fire-and-limit concurrency via Task.Run? Better: use a SemaphoreSlim |
| | 3 | 43 | | _ = ProcessOne(req, stoppingToken); |
| | | 44 | | } |
| | 0 | 45 | | } |
| | 2 | 46 | | catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) |
| | | 47 | | { |
| | | 48 | | // Graceful shutdown |
| | 2 | 49 | | if (_log.IsEnabled(Serilog.Events.LogEventLevel.Information)) |
| | | 50 | | { |
| | 2 | 51 | | _log.Information("CallbackWorker is stopping due to cancellation."); |
| | | 52 | | } |
| | 2 | 53 | | } |
| | 2 | 54 | | } |
| | | 55 | | |
| | | 56 | | /// <summary> |
| | | 57 | | /// Processes a single callback request. |
| | | 58 | | /// </summary> |
| | | 59 | | /// <param name="req"> The callback request to process.</param> |
| | | 60 | | /// <param name="ct"> The cancellation token.</param> |
| | | 61 | | /// <returns> A task that represents the asynchronous operation.</returns> |
| | | 62 | | private async Task ProcessOne(CallbackRequest req, CancellationToken ct) |
| | | 63 | | { |
| | | 64 | | try |
| | | 65 | | { |
| | 3 | 66 | | if (_store is not null) |
| | | 67 | | { |
| | 3 | 68 | | await _store.MarkInFlightAsync(req, ct); |
| | | 69 | | } |
| | | 70 | | |
| | 3 | 71 | | var result = await _sender.SendAsync(req, ct); |
| | | 72 | | |
| | 3 | 73 | | if (result.Success) |
| | | 74 | | { |
| | 1 | 75 | | if (_store is not null) |
| | | 76 | | { |
| | 1 | 77 | | await _store.MarkSucceededAsync(req, result, ct); |
| | | 78 | | } |
| | | 79 | | |
| | 1 | 80 | | return; |
| | | 81 | | } |
| | | 82 | | |
| | 2 | 83 | | await HandleFailure(req, result, ct); |
| | 2 | 84 | | } |
| | 0 | 85 | | catch (Exception ex) |
| | | 86 | | { |
| | 0 | 87 | | var result = new CallbackResult(false, null, ex.GetType().Name, ex.Message, DateTimeOffset.UtcNow); |
| | 0 | 88 | | await HandleFailure(req, result, ct); |
| | | 89 | | } |
| | 3 | 90 | | } |
| | | 91 | | |
| | | 92 | | /// <summary> |
| | | 93 | | /// Handles a failed callback request. |
| | | 94 | | /// </summary> |
| | | 95 | | /// <param name="req"> The callback request that failed.</param> |
| | | 96 | | /// <param name="result"> The result of the callback attempt.</param> |
| | | 97 | | /// <param name="ct"> The cancellation token.</param> |
| | | 98 | | /// <returns> A task that represents the asynchronous operation.</returns> |
| | | 99 | | private async Task HandleFailure(CallbackRequest req, CallbackResult result, CancellationToken ct) |
| | | 100 | | { |
| | 2 | 101 | | var decision = _retry.Evaluate(req, result); |
| | | 102 | | |
| | 2 | 103 | | if (decision.Kind == RetryDecisionKind.Retry) |
| | | 104 | | { |
| | 1 | 105 | | req.Attempt++; |
| | 1 | 106 | | req.NextAttemptAt = decision.NextAttemptAt; |
| | | 107 | | |
| | 1 | 108 | | if (_store is not null) |
| | | 109 | | { |
| | 1 | 110 | | await _store.MarkRetryScheduledAsync(req, result, ct); |
| | | 111 | | } |
| | | 112 | | |
| | | 113 | | // schedule re-enqueue (simple in-memory) or rely on durable poller |
| | 1 | 114 | | _ = Task.Delay(decision.Delay, ct).ContinueWith(async _ => |
| | 1 | 115 | | { |
| | 1 | 116 | | // ignore exceptions if shutting down |
| | 2 | 117 | | try { await EnqueueAgain(req, ct); } |
| | 0 | 118 | | catch (OperationCanceledException) |
| | 1 | 119 | | { |
| | 1 | 120 | | // expected during shutdown; no further action required |
| | 0 | 121 | | if (_log.IsEnabled(Serilog.Events.LogEventLevel.Debug)) |
| | 1 | 122 | | { |
| | 0 | 123 | | _log.Debug("Callback {CallbackId} re-enqueue skipped due to shutdown.", |
| | 0 | 124 | | req.CallbackId); |
| | 1 | 125 | | } |
| | 0 | 126 | | } |
| | 0 | 127 | | catch (Exception ex) |
| | 1 | 128 | | { |
| | 0 | 129 | | if (_log.IsEnabled(Serilog.Events.LogEventLevel.Debug)) |
| | 1 | 130 | | { |
| | 0 | 131 | | _log.Debug(ex, |
| | 0 | 132 | | "Failed to re-enqueue callback {CallbackId} during retry scheduling.", |
| | 0 | 133 | | req.CallbackId); |
| | 1 | 134 | | } |
| | 0 | 135 | | } |
| | 2 | 136 | | }, TaskScheduler.Default); |
| | | 137 | | |
| | 1 | 138 | | return; |
| | | 139 | | } |
| | | 140 | | |
| | 1 | 141 | | if (_store is not null) |
| | | 142 | | { |
| | 1 | 143 | | await _store.MarkFailedPermanentAsync(req, result, ct); |
| | | 144 | | } |
| | | 145 | | |
| | 1 | 146 | | _log.Warning("Callback failed permanently {CallbackId} after {Attempts} attempts. Last error: {Err}", |
| | 1 | 147 | | req.CallbackId, req.Attempt + 1, result.ErrorMessage); |
| | 2 | 148 | | } |
| | | 149 | | |
| | | 150 | | /// <summary> |
| | | 151 | | /// Enqueues the callback request again for retry. |
| | | 152 | | /// </summary> |
| | | 153 | | /// <param name="req"> The callback request to enqueue again.</param> |
| | | 154 | | /// <param name="ct"> The cancellation token.</param> |
| | | 155 | | /// <returns> A task that represents the asynchronous operation.</returns> |
| | | 156 | | private async Task EnqueueAgain(CallbackRequest req, CancellationToken ct) |
| | 1 | 157 | | => await _writer.WriteAsync(req, ct).ConfigureAwait(false); |
| | | 158 | | } |