4102568c88
* Per resource webhook queue. With a worker based approach, an event on a resource (like a room) not being handled could affect other resources (other rooms or ingress or egress). To prevent that, do a per resource queue with two kinds of limits - age of event: when queuing, if the event at the front of the queue exceeds a certain limit (default: 5 seconds), the event is dropped - depth of queue: drop event if the number of queued events exceeds some level (default: 200) To reap the per resource queue, specific end events are tracked and the resource is added to the reap queue and reaped after a configurable timeout (default: 5 min). This is to ensure that events happening just after end event (for example: participant_left happening after room_finished) are handled without re-creating a new resource. Also, deprecating `num_dropped` from proto. Some other bits of notes/challenges: - Keeping the old URL notifier also. Also, the default notifier is that. Can change to the new one before merging this PR if this approach looks good. - Using string compares of event name to detect end. Not great. - Needs a sweeper to reap the resource queue. I think 5 min default timeout to reap should be okay, but maybe there are things that can cause re-creation of a resource queue after reaping and those could leak? Needs more analysis. - Storing `context.Context` in resource queue so that it can be used during posting to webhook from resource queue go routine. Storing that is not great, but we need it for analytics key. * generated protobuf * missed files * generated protobuf * fix test in data race * generate * generated protobuf * paul feedback * fixes * race * more Paul feedback * make resource notifier the default * generated protobuf * stop --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
136 lines
2.5 KiB
Go
136 lines
2.5 KiB
Go
// Copyright 2023 LiveKit, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package webhook
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/gammazero/deque"
|
|
"github.com/livekit/protocol/livekit"
|
|
)
|
|
|
|
var (
|
|
errQueueFull = errors.New("queue is full")
|
|
errQueueClosed = errors.New("queue is closed")
|
|
)
|
|
|
|
type item struct {
|
|
ctx context.Context
|
|
queuedAt time.Time
|
|
event *livekit.WebhookEvent
|
|
}
|
|
|
|
type resourceQueueParams struct {
|
|
MaxDepth int
|
|
|
|
Poster poster
|
|
}
|
|
|
|
type resourceQueue struct {
|
|
params resourceQueueParams
|
|
|
|
mu sync.Mutex
|
|
items deque.Deque[*item]
|
|
cond *sync.Cond
|
|
|
|
closed bool
|
|
drain bool
|
|
}
|
|
|
|
func newResourceQueue(params resourceQueueParams) *resourceQueue {
|
|
r := &resourceQueue{
|
|
params: params,
|
|
}
|
|
r.items.SetBaseCap(int(min(params.MaxDepth, 16)))
|
|
r.cond = sync.NewCond(&r.mu)
|
|
|
|
go r.worker()
|
|
return r
|
|
}
|
|
|
|
func (r *resourceQueue) Stop(force bool) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
if !r.closed {
|
|
r.closed = true
|
|
r.drain = !force
|
|
|
|
r.cond.Broadcast()
|
|
}
|
|
}
|
|
|
|
func (r *resourceQueue) Enqueue(ctx context.Context, whEvent *livekit.WebhookEvent) error {
|
|
return r.EnqueueAt(ctx, time.Now(), whEvent)
|
|
}
|
|
|
|
func (r *resourceQueue) EnqueueAt(ctx context.Context, at time.Time, whEvent *livekit.WebhookEvent) error {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
if r.closed {
|
|
return errQueueClosed
|
|
}
|
|
|
|
if r.items.Len() >= r.params.MaxDepth {
|
|
return errQueueFull
|
|
}
|
|
|
|
r.items.PushBack(&item{ctx, at, whEvent})
|
|
r.cond.Broadcast()
|
|
return nil
|
|
}
|
|
|
|
func (r *resourceQueue) flush() {
|
|
r.mu.Lock()
|
|
for r.items.Len() > 0 {
|
|
item := r.items.PopFront()
|
|
r.mu.Unlock()
|
|
|
|
r.params.Poster.Process(item.ctx, item.queuedAt, item.event)
|
|
|
|
r.mu.Lock()
|
|
}
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
func (r *resourceQueue) worker() {
|
|
for {
|
|
r.mu.Lock()
|
|
for {
|
|
if r.closed {
|
|
r.mu.Unlock()
|
|
if r.drain {
|
|
r.flush()
|
|
}
|
|
return
|
|
}
|
|
|
|
if r.items.Len() != 0 {
|
|
break
|
|
}
|
|
r.cond.Wait()
|
|
}
|
|
|
|
item := r.items.PopFront()
|
|
r.mu.Unlock()
|
|
|
|
r.params.Poster.Process(item.ctx, item.queuedAt, item.event)
|
|
}
|
|
}
|