mirror of
https://github.com/dagu-org/dagu.git
synced 2025-12-27 22:26:13 +00:00
Compliance with XDG (#619)
This commit is contained in:
parent
3a6bf29bfe
commit
38695a2f86
6
.gitignore
vendored
6
.gitignore
vendored
@ -19,7 +19,5 @@ tmp/*
|
||||
# Goland
|
||||
.idea
|
||||
|
||||
# local development
|
||||
!local/cert
|
||||
local/cert/*
|
||||
!local/cert/openssl.conf
|
||||
# Directory for local development
|
||||
local/
|
||||
|
||||
@ -45,10 +45,10 @@ WORKDIR /home/${USER}
|
||||
|
||||
COPY --from=go-builder /app/bin/dagu /usr/local/bin/
|
||||
|
||||
RUN mkdir -p .dagu/dags
|
||||
RUN mkdir -p .config/dagu/dags
|
||||
|
||||
# Add the hello_world.yaml file
|
||||
COPY <<EOF .dagu/dags/hello_world.yaml
|
||||
COPY <<EOF .config/dagu/dags/hello_world.yaml
|
||||
schedule: "* * * * *"
|
||||
steps:
|
||||
- name: hello world
|
||||
|
||||
191
Makefile
191
Makefile
@ -1,99 +1,130 @@
|
||||
.PHONY: build server scheduler test proto certs swagger https
|
||||
.PHONY: run run-server run-server-https run-scheduler test lint build certs swagger
|
||||
|
||||
########## Arguments ##########
|
||||
##############################################################################
|
||||
# Arguments
|
||||
##############################################################################
|
||||
|
||||
VERSION=
|
||||
|
||||
########## Variables ##########
|
||||
##############################################################################
|
||||
# Variables
|
||||
##############################################################################
|
||||
|
||||
# This Makefile's directory
|
||||
SCRIPT_DIR=$(abspath $(dir $(lastword $(MAKEFILE_LIST))))
|
||||
|
||||
# Directories for miscellaneous files for the local environment
|
||||
LOCAL_DIR=$(SCRIPT_DIR)/local
|
||||
LOCAL_BIN_DIR=$(LOCAL_DIR)/bin
|
||||
|
||||
SRC_DIR=$(SCRIPT_DIR)
|
||||
DST_DIR=$(SRC_DIR)/internal
|
||||
# Configuration directory
|
||||
CONFIG_DIR=$(SCRIPT_DIR)/config
|
||||
|
||||
# Local build settings
|
||||
BIN_DIR=$(SCRIPT_DIR)/bin
|
||||
BUILD_VERSION=$(shell date +'%y%m%d%H%M%S')
|
||||
LDFLAGS=-X 'main.version=$(BUILD_VERSION)'
|
||||
|
||||
# Application name
|
||||
|
||||
APP_NAME=dagu
|
||||
|
||||
# Docker image build configuration
|
||||
DOCKER_CMD := docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7,linux/arm64/v8 --builder container --build-arg VERSION=$(VERSION) --push --no-cache
|
||||
|
||||
# Arguments for the tests
|
||||
GOTESTSUM_ARGS=--format=standard-quiet
|
||||
GO_TEST_FLAGS=-v --race
|
||||
|
||||
# Frontend directories
|
||||
|
||||
FE_DIR=./internal/frontend
|
||||
FE_GEN_DIR=${FE_DIR}/gen
|
||||
FE_ASSETS_DIR=${FE_DIR}/assets
|
||||
FE_BUILD_DIR=./ui/dist
|
||||
FE_BUNDLE_JS=${FE_ASSETS_DIR}/bundle.js
|
||||
|
||||
# Colors for the output
|
||||
|
||||
COLOR_GREEN=\033[0;32m
|
||||
COLOR_RESET=\033[0m
|
||||
COLOR_RED=\033[0;31m
|
||||
|
||||
# Go packages for the tools
|
||||
|
||||
PKG_swagger=github.com/go-swagger/go-swagger/cmd/swagger
|
||||
PKG_golangci_lint=github.com/golangci/golangci-lint/cmd/golangci-lint
|
||||
PKG_gotestsum=gotest.tools/gotestsum
|
||||
PKG_gomerger=github.com/yohamta/gomerger
|
||||
|
||||
# Certificates for the development environment
|
||||
|
||||
CERTS_DIR=${LOCAL_DIR}/certs
|
||||
|
||||
DEV_CERT_SUBJ_CA="/C=TR/ST=ASIA/L=TOKYO/O=DEV/OU=DAGU/CN=*.dagu.dev/emailAddress=ca@dev.com"
|
||||
DEV_CERT_SUBJ_SERVER="/C=TR/ST=ASIA/L=TOKYO/O=DEV/OU=SERVER/CN=*.server.dev/emailAddress=server@dev.com"
|
||||
DEV_CERT_SUBJ_CLIENT="/C=TR/ST=ASIA/L=TOKYO/O=DEV/OU=CLIENT/CN=*.client.dev/emailAddress=client@dev.com"
|
||||
DEV_CERT_SUBJ_ALT="subjectAltName=DNS:localhost"
|
||||
|
||||
PKG_SWAGGER=github.com/go-swagger/go-swagger/cmd/swagger
|
||||
PKG_GOLANGCI_LINT=github.com/golangci/golangci-lint/cmd/golangci-lint
|
||||
PKG_gotestsum=gotest.tools/gotestsum
|
||||
CA_CERT_FILE=${CERTS_DIR}/ca-cert.pem
|
||||
CA_KEY_FILE=${CERTS_DIR}/ca-key.pem
|
||||
SERVER_CERT_REQ=${CERTS_DIR}/server-req.pem
|
||||
SERVER_CERT_FILE=${CERTS_DIR}/server-cert.pem
|
||||
SERVER_KEY_FILE=${CERTS_DIR}/server-key.pem
|
||||
CLIENT_CERT_REQ=${CERTS_DIR}/client-req.pem
|
||||
CLIENT_CERT_FILE=${CERTS_DIR}/client-cert.pem
|
||||
CLIENT_KEY_FILE=${CERTS_DIR}/client-key.pem
|
||||
OPENSSL_CONF=${CONFIG_DIR}/openssl.local.conf
|
||||
|
||||
COLOR_GREEN=\033[0;32m
|
||||
COLOR_RESET=\033[0m
|
||||
|
||||
FE_DIR=./internal/frontend
|
||||
FE_GEN_DIR=${FE_DIR}/gen
|
||||
FE_ASSETS_DIR=${FE_DIR}/assets
|
||||
|
||||
CERT_DIR=${LOCAL_DIR}/cert
|
||||
|
||||
CA_CERT_FILE=${CERT_DIR}/ca-cert.pem
|
||||
CA_KEY_FILE=${CERT_DIR}/ca-key.pem
|
||||
SERVER_CERT_REQ=${CERT_DIR}/server-req.pem
|
||||
SERVER_CERT_FILE=${CERT_DIR}/server-cert.pem
|
||||
SERVER_KEY_FILE=${CERT_DIR}/server-key.pem
|
||||
CLIENT_CERT_REQ=${CERT_DIR}/client-req.pem
|
||||
CLIENT_CERT_FILE=${CERT_DIR}/client-cert.pem
|
||||
CLIENT_KEY_FILE=${CERT_DIR}/client-key.pem
|
||||
OPENSSL_CONF=${CERT_DIR}/openssl.conf
|
||||
|
||||
FE_BUILD_DIR=./ui/dist
|
||||
FE_BUNDLE_JS=${FE_ASSETS_DIR}/bundle.js
|
||||
|
||||
APP_NAME=dagu
|
||||
BIN_DIR=${SCRIPT_DIR}/bin
|
||||
|
||||
# gotestsum args
|
||||
GOTESTSUM_ARGS=--format=standard-quiet
|
||||
GO_TEST_FLAGS=-v --race
|
||||
|
||||
########## Main Targets ##########
|
||||
##############################################################################
|
||||
# Targets
|
||||
##############################################################################
|
||||
|
||||
# run starts the frontend server and the scheduler.
|
||||
run: ${FE_BUNDLE_JS}
|
||||
go run . start-all
|
||||
@echo "${COLOR_GREEN}Starting the frontend server and the scheduler...${COLOR_RESET}"
|
||||
@go run . start-all
|
||||
|
||||
# server build the binary and start the server.
|
||||
run-server: golangci-lint build-bin
|
||||
@echo "${COLOR_GREEN}Starting the server...${COLOR_RESET}"
|
||||
${LOCAL_BIN_DIR}/${APP_NAME} server
|
||||
|
||||
# scheduler build the binary and start the scheduler.
|
||||
run-scheduler: golangci-lint build-bin
|
||||
@echo "${COLOR_GREEN}Starting the scheduler...${COLOR_RESET}"
|
||||
${LOCAL_BIN_DIR}/${APP_NAME} scheduler
|
||||
|
||||
# check if the frontend assets are built.
|
||||
${FE_BUNDLE_JS}:
|
||||
echo "Please run 'make build-ui' to build the frontend assets."
|
||||
@echo "${COLOR_RED}Error: frontend assets are not built.${COLOR_RESET}"
|
||||
@echo "${COLOR_RED}Please run 'make build-ui' before starting the server.${COLOR_RESET}"
|
||||
|
||||
# https starts the server with the HTTPS protocol.
|
||||
https: ${SERVER_CERT_FILE} ${SERVER_KEY_FILE}
|
||||
run-server-https: ${SERVER_CERT_FILE} ${SERVER_KEY_FILE}
|
||||
@echo "${COLOR_GREEN}Starting the server with HTTPS...${COLOR_RESET}"
|
||||
@DAGU_CERT_FILE=${SERVER_CERT_FILE} \
|
||||
DAGU_KEY_FILE=${SERVER_KEY_FILE} \
|
||||
go run . start-all
|
||||
|
||||
# watch starts development UI server.
|
||||
# The backend server should be running.
|
||||
watch:
|
||||
@echo "${COLOR_GREEN}Installing nodemon...${COLOR_RESET}"
|
||||
@npm install -g nodemon
|
||||
@nodemon --watch . --ext go,gohtml --verbose --signal SIGINT --exec 'make server'
|
||||
|
||||
# test runs all tests.
|
||||
test:
|
||||
@go install ${PKG_gotestsum}
|
||||
@gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} ./...
|
||||
@echo "${COLOR_GREEN}Running tests...${COLOR_RESET}"
|
||||
@GOBIN=${LOCAL_BIN_DIR} go install ${PKG_gotestsum}
|
||||
@${LOCAL_BIN_DIR}/gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} ./...
|
||||
|
||||
# test-coverage runs all tests with coverage.
|
||||
test-coverage:
|
||||
@go install ${PKG_gotestsum}
|
||||
@gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} -coverprofile="coverage.txt" -covermode=atomic ./...
|
||||
@echo "${COLOR_GREEN}Running tests with coverage...${COLOR_RESET}"
|
||||
@GOBIN=${LOCAL_BIN_DIR} go install ${PKG_gotestsum}
|
||||
@${LOCAL_BIN_DIR}/gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} -coverprofile="coverage.txt" -covermode=atomic ./...
|
||||
|
||||
# test-clean cleans the test cache and run all tests.
|
||||
test-clean: build-bin
|
||||
@go install ${PKG_gotestsum}
|
||||
@echo "${COLOR_GREEN}Running tests...${COLOR_RESET}"
|
||||
@GOBIN=${LOCAL_BIN_DIR} go install ${PKG_gotestsum}
|
||||
@go clean -testcache
|
||||
@gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} ./...
|
||||
@${LOCAL_BIN_DIR}/gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} ./...
|
||||
|
||||
# lint runs the linter.
|
||||
lint: golangci-lint
|
||||
@ -102,7 +133,7 @@ lint: golangci-lint
|
||||
swagger: clean-swagger gen-swagger
|
||||
|
||||
# certs generates the certificates to use in the development environment.
|
||||
certs: ${SERVER_CERT_FILE} ${CLIENT_CERT_FILE} gencert-check
|
||||
certs: ${CERTS_DIR} ${SERVER_CERT_FILE} ${CLIENT_CERT_FILE} certs-check
|
||||
|
||||
# build build the binary.
|
||||
build: build-ui build-bin
|
||||
@ -112,30 +143,38 @@ build: build-ui build-bin
|
||||
# ```sh
|
||||
# make build-image VERSION={version}
|
||||
# ```
|
||||
# {version} should be the version number such as v1.13.0.
|
||||
# {version} should be the version number such as "1.13.0".
|
||||
|
||||
build-image: build-image-version build-image-latest
|
||||
build-image-version:
|
||||
ifeq ($(VERSION),)
|
||||
$(error "VERSION is null")
|
||||
$(error "VERSION is not set")
|
||||
endif
|
||||
echo "${COLOR_GREEN}Building the docker image with the version $(VERSION)...${COLOR_RESET}"
|
||||
$(DOCKER_CMD) -t ghcr.io/dagu-dev/${APP_NAME}:$(VERSION) .
|
||||
|
||||
# build-image-latest build the docker image with the latest tag and push to
|
||||
# the registry.
|
||||
build-image-latest:
|
||||
@echo "${COLOR_GREEN}Building the docker image...${COLOR_RESET}"
|
||||
$(DOCKER_CMD) -t ghcr.io/dagu-dev/${APP_NAME}:latest .
|
||||
|
||||
# server build the binary and start the server.
|
||||
server: golangci-lint build-bin
|
||||
${BIN_DIR}/${APP_NAME} server
|
||||
gomerger: ${LOCAL_DIR}/merged
|
||||
@echo "${COLOR_GREEN}Merging Go files...${COLOR_RESET}"
|
||||
@rm -f ${LOCAL_DIR}/merged/merged_project.go
|
||||
@GOBIN=${LOCAL_BIN_DIR} go install ${PKG_gomerger}
|
||||
@${LOCAL_BIN_DIR}/gomerger .
|
||||
@mv merged_project.go ${LOCAL_DIR}/merged/
|
||||
|
||||
# scheduler build the binary and start the scheduler.
|
||||
scheduler: golangci-lint build-bin
|
||||
${BIN_DIR}/${APP_NAME} scheduler
|
||||
${LOCAL_DIR}/merged:
|
||||
@mkdir -p ${LOCAL_DIR}/merged
|
||||
|
||||
########## Tools ##########
|
||||
##############################################################################
|
||||
# Internal targets
|
||||
##############################################################################
|
||||
|
||||
build-bin:
|
||||
@echo "${COLOR_GREEN}Building the binary...${COLOR_RESET}"
|
||||
@mkdir -p ${BIN_DIR}
|
||||
@go build -ldflags="$(LDFLAGS)" -o ${BIN_DIR}/${APP_NAME} .
|
||||
|
||||
@ -148,20 +187,25 @@ build-ui:
|
||||
@cp ${FE_BUILD_DIR}/* ${FE_ASSETS_DIR}
|
||||
|
||||
golangci-lint:
|
||||
@go install $(PKG_GOLANGCI_LINT)
|
||||
@golangci-lint run ./...
|
||||
@echo "${COLOR_GREEN}Running linter...${COLOR_RESET}"
|
||||
@GOBIN=${LOCAL_BIN_DIR} go install $(PKG_golangci_lint)
|
||||
@${LOCAL_BIN_DIR}/golangci-lint run ./...
|
||||
|
||||
clean-swagger:
|
||||
@echo "${COLOR_GREEN}Cleaning the swagger files...${COLOR_RESET}"
|
||||
@rm -rf ${FE_GEN_DIR}/restapi/models
|
||||
@rm -rf ${FE_GEN_DIR}/restapi/operations
|
||||
|
||||
gen-swagger:
|
||||
@go install $(PKG_SWAGGER)
|
||||
@swagger validate ./swagger.yaml
|
||||
@swagger generate server -t ${FE_GEN_DIR} --server-package=restapi --exclude-main -f ./swagger.yaml
|
||||
@echo "${COLOR_GREEN}Generating the swagger server code...${COLOR_RESET}"
|
||||
@GOBIN=${LOCAL_BIN_DIR} go install $(PKG_swagger)
|
||||
@${LOCAL_BIN_DIR}/swagger validate ./swagger.yaml
|
||||
@${LOCAL_BIN_DIR}/swagger generate server -t ${FE_GEN_DIR} --server-package=restapi --exclude-main -f ./swagger.yaml
|
||||
@go mod tidy
|
||||
|
||||
########## Certificates ##########
|
||||
##############################################################################
|
||||
# Certificates
|
||||
##############################################################################
|
||||
|
||||
${CA_CERT_FILE}:
|
||||
@echo "${COLOR_GREEN}Generating CA certificates...${COLOR_RESET}"
|
||||
@ -194,7 +238,10 @@ ${CLIENT_CERT_FILE}: ${CA_CERT_FILE} ${CLIENT_KEY_FILE}
|
||||
-CAkey ${CA_KEY_FILE} -CAcreateserial -out ${CLIENT_CERT_FILE} \
|
||||
-extfile ${OPENSSL_CONF}
|
||||
|
||||
gencert-check:
|
||||
${CERTS_DIR}:
|
||||
@echo "${COLOR_GREEN}Creating the certificates directory...${COLOR_RESET}"
|
||||
@mkdir -p ${CERTS_DIR}
|
||||
|
||||
certs-check:
|
||||
@echo "${COLOR_GREEN}Checking CA certificate...${COLOR_RESET}"
|
||||
@openssl x509 -in ${SERVER_CERT_FILE} -noout -text
|
||||
|
||||
|
||||
14
README.md
14
README.md
@ -67,6 +67,7 @@ Dagu is a powerful Cron alternative that comes with a Web UI. It allows you to d
|
||||
- [**Documentation**](#documentation)
|
||||
- [**Running as a daemon**](#running-as-a-daemon)
|
||||
- [**Example DAG**](#example-dag)
|
||||
- [**Docker-compose setting**](#docker-compose-setting)
|
||||
- [**Motivation**](#motivation)
|
||||
- [**Why Not Use an Existing DAG Scheduler Like Airflow?**](#why-not-use-an-existing-dag-scheduler-like-airflow)
|
||||
- [**How It Works**](#how-it-works)
|
||||
@ -176,12 +177,13 @@ brew upgrade dagu-dev/brew/dagu
|
||||
docker run \
|
||||
--rm \
|
||||
-p 8080:8080 \
|
||||
-v $HOME/.dagu/dags:/home/dagu/.dagu/dags \
|
||||
-v $HOME/.dagu/data:/home/dagu/.dagu/data \
|
||||
-v $HOME/.dagu/logs:/home/dagu/.dagu/logs \
|
||||
-v $HOME/.config/dagu/dags:/home/dagu/.config/dagu/dags \
|
||||
-v $HOME/.local/share/dagu:/home/dagu/.local/share/dagu \
|
||||
ghcr.io/dagu-dev/dagu:latest dagu start-all
|
||||
```
|
||||
|
||||
See [Environment variables](https://dagu.readthedocs.io/en/latest/config.html#environment-variables) to configure those default directories.
|
||||
|
||||
## **Quick Start Guide**
|
||||
|
||||
### 1. Launch the Web UI
|
||||
@ -192,7 +194,7 @@ Start the server and scheduler with the command `dagu start-all` and browse to `
|
||||
|
||||
Navigate to the DAG List page by clicking the menu in the left panel of the Web UI. Then create a DAG by clicking the `NEW` button at the top of the page. Enter `example` in the dialog.
|
||||
|
||||
_Note: DAG (YAML) files will be placed in `~/.dagu/dags` by default. See [Configuration Options](https://dagu.readthedocs.io/en/latest/config.html) for more details._
|
||||
_Note: DAG (YAML) files will be placed in `~/.config/dagu/dags` by default. See [Configuration Options](https://dagu.readthedocs.io/en/latest/config.html) for more details._
|
||||
|
||||
### 3. Edit the DAG
|
||||
|
||||
@ -387,6 +389,10 @@ steps:
|
||||
- send_report
|
||||
```
|
||||
|
||||
## **Docker-compose setting**
|
||||
|
||||
To run Dagu using Docker-compose, please take a look at the example: [docker-compose file](examples/docker-compose.yaml)
|
||||
|
||||
## **Motivation**
|
||||
|
||||
Legacy systems often have complex and implicit dependencies between jobs. When there are hundreds of cron jobs on a server, it can be difficult to keep track of these dependencies and to determine which job to rerun if one fails. It can also be a hassle to SSH into a server to view logs and manually rerun shell scripts one by one. Dagu aims to solve these problems by allowing you to explicitly visualize and manage pipeline dependencies as a DAG, and by providing a web UI for checking dependencies, execution status, and logs and for rerunning or stopping jobs with a simple mouse click.
|
||||
|
||||
@ -1,8 +1,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -17,43 +16,6 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type testSetup struct {
|
||||
homeDir string
|
||||
engine engine.Engine
|
||||
dataStore persistence.DataStores
|
||||
cfg *config.Config
|
||||
}
|
||||
|
||||
func (t testSetup) cleanup() {
|
||||
_ = os.RemoveAll(t.homeDir)
|
||||
}
|
||||
|
||||
// setupTest is a helper function to setup the test environment.
|
||||
// This function does the following:
|
||||
// 1. It creates a temporary directory and returns the path to it.
|
||||
// 2. Sets the home directory to the temporary directory.
|
||||
// 3. Creates a new data store factory and engine.
|
||||
func setupTest(t *testing.T) testSetup {
|
||||
t.Helper()
|
||||
|
||||
tmpDir := util.MustTempDir("dagu_test")
|
||||
err := os.Setenv("HOME", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg, err := config.Load()
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg.DataDir = path.Join(tmpDir, ".dagu", "data")
|
||||
dataStore := newDataStores(cfg)
|
||||
|
||||
return testSetup{
|
||||
homeDir: tmpDir,
|
||||
dataStore: dataStore,
|
||||
engine: newEngine(cfg),
|
||||
cfg: cfg,
|
||||
}
|
||||
}
|
||||
|
||||
// cmdTest is a helper struct to test commands.
|
||||
// It contains the arguments to the command and the expected output.
|
||||
type cmdTest struct {
|
||||
@ -128,8 +90,8 @@ func withSpool(t *testing.T, testFunction func()) string {
|
||||
*/
|
||||
|
||||
func testDAGFile(name string) string {
|
||||
return path.Join(
|
||||
path.Join(util.MustGetwd(), "testdata"),
|
||||
return filepath.Join(
|
||||
filepath.Join(util.MustGetwd(), "testdata"),
|
||||
name,
|
||||
)
|
||||
}
|
||||
|
||||
@ -2,16 +2,18 @@ package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/test"
|
||||
)
|
||||
|
||||
func TestDryCommand(t *testing.T) {
|
||||
t.Run("Dry-run command should run", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("DryRun", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
tests := []cmdTest{
|
||||
{
|
||||
args: []string{"dry", testDAGFile("dry.yaml")},
|
||||
args: []string{"dry", testDAGFile("success.yaml")},
|
||||
expectedOut: []string{"Starting DRY-RUN"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@ import (
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/dag"
|
||||
"github.com/dagu-dev/dagu/internal/dag/scheduler"
|
||||
"github.com/dagu-dev/dagu/internal/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@ -14,9 +15,9 @@ const (
|
||||
)
|
||||
|
||||
func TestRestartCommand(t *testing.T) {
|
||||
t.Run("Restart a DAG", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("RestartDAG", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
dagFile := testDAGFile("restart.yaml")
|
||||
|
||||
@ -30,7 +31,7 @@ func TestRestartCommand(t *testing.T) {
|
||||
}()
|
||||
|
||||
time.Sleep(waitForStatusUpdate)
|
||||
eng := setup.engine
|
||||
eng := setup.Engine()
|
||||
|
||||
// Wait for the DAG running.
|
||||
testStatusEventual(t, eng, dagFile, scheduler.StatusRunning)
|
||||
@ -56,10 +57,10 @@ func TestRestartCommand(t *testing.T) {
|
||||
testStatusEventual(t, eng, dagFile, scheduler.StatusNone)
|
||||
|
||||
// Check parameter was the same as the first execution
|
||||
dg, err := dag.Load(setup.cfg.BaseConfig, dagFile, "")
|
||||
dg, err := dag.Load(setup.Config.BaseConfig, dagFile, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
recentHistory := newEngine(setup.cfg).GetRecentHistory(dg, 2)
|
||||
recentHistory := newEngine(setup.Config).GetRecentHistory(dg, 2)
|
||||
|
||||
require.Len(t, recentHistory, 2)
|
||||
require.Equal(t, recentHistory[0].Status.Params, recentHistory[1].Status.Params)
|
||||
|
||||
@ -5,13 +5,14 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/dag/scheduler"
|
||||
"github.com/dagu-dev/dagu/internal/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRetryCommand(t *testing.T) {
|
||||
t.Run("Retry a DAG", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("RetryDAG", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
dagFile := testDAGFile("retry.yaml")
|
||||
|
||||
@ -19,7 +20,7 @@ func TestRetryCommand(t *testing.T) {
|
||||
testRunCommand(t, startCmd(), cmdTest{args: []string{"start", `--params="foo"`, dagFile}})
|
||||
|
||||
// Find the request ID.
|
||||
eng := setup.engine
|
||||
eng := setup.Engine()
|
||||
status, err := eng.GetStatus(dagFile)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, status.Status.Status, scheduler.StatusSuccess)
|
||||
|
||||
18
cmd/root.go
18
cmd/root.go
@ -4,9 +4,7 @@ Copyright © 2023 NAME HERE <EMAIL ADDRESS>
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/config"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
@ -23,8 +21,6 @@ var (
|
||||
}
|
||||
)
|
||||
|
||||
const configPath = ".dagu"
|
||||
|
||||
// Execute adds all child commands to the root command and sets flags
|
||||
// appropriately. This is called by main.main(). It only needs to happen
|
||||
// once to the rootCmd.
|
||||
@ -32,14 +28,6 @@ func Execute() error {
|
||||
return rootCmd.Execute()
|
||||
}
|
||||
|
||||
func setDefaultConfigPath() {
|
||||
homeDir, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
panic("could not determine home directory")
|
||||
}
|
||||
viper.AddConfigPath(path.Join(homeDir, configPath))
|
||||
}
|
||||
|
||||
func registerCommands() {
|
||||
rootCmd.AddCommand(startCmd())
|
||||
rootCmd.AddCommand(stopCmd())
|
||||
@ -57,7 +45,7 @@ func init() {
|
||||
rootCmd.PersistentFlags().
|
||||
StringVar(
|
||||
&cfgFile, "config", "",
|
||||
"config file (default is $HOME/.dagu/admin.yaml)",
|
||||
"config file (default is $HOME/.config/dagu/admin.yaml)",
|
||||
)
|
||||
|
||||
cobra.OnInitialize(initialize)
|
||||
@ -71,7 +59,7 @@ func initialize() {
|
||||
return
|
||||
}
|
||||
|
||||
setDefaultConfigPath()
|
||||
viper.AddConfigPath(config.ConfigDir)
|
||||
viper.SetConfigType("yaml")
|
||||
viper.SetConfigName("admin")
|
||||
}
|
||||
|
||||
@ -39,7 +39,7 @@ func schedulerCmd() *cobra.Command {
|
||||
}
|
||||
|
||||
cmd.Flags().StringP(
|
||||
"dags", "d", "", "location of DAG files (default is $HOME/.dagu/dags)",
|
||||
"dags", "d", "", "location of DAG files (default is $HOME/.config/dagu/dags)",
|
||||
)
|
||||
_ = viper.BindPFlag("dags", cmd.Flags().Lookup("dags"))
|
||||
|
||||
|
||||
@ -3,12 +3,14 @@ package cmd
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/test"
|
||||
)
|
||||
|
||||
func TestSchedulerCommand(t *testing.T) {
|
||||
t.Run("Start the scheduler", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("StartScheduler", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
go func() {
|
||||
testRunCommand(t, schedulerCmd(), cmdTest{
|
||||
|
||||
@ -47,7 +47,7 @@ func serverCmd() *cobra.Command {
|
||||
|
||||
func bindServerCommandFlags(cmd *cobra.Command) {
|
||||
cmd.Flags().StringP(
|
||||
"dags", "d", "", "location of DAG files (default is $HOME/.dagu/dags)",
|
||||
"dags", "d", "", "location of DAG files (default is $HOME/.config/dagu/dags)",
|
||||
)
|
||||
cmd.Flags().StringP("host", "s", "", "server host (default is localhost)")
|
||||
cmd.Flags().StringP("port", "p", "", "server port (default is 8080)")
|
||||
|
||||
@ -6,13 +6,14 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestServerCommand(t *testing.T) {
|
||||
t.Run("Start the server", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("StartServer", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
go func() {
|
||||
testRunCommand(t, serverCmd(), cmdTest{
|
||||
|
||||
@ -69,7 +69,7 @@ func startAllCmd() *cobra.Command {
|
||||
|
||||
func bindStartAllCommandFlags(cmd *cobra.Command) {
|
||||
cmd.Flags().StringP(
|
||||
"dags", "d", "", "location of DAG files (default is $HOME/.dagu/dags)",
|
||||
"dags", "d", "", "location of DAG files (default is $HOME/.config/dagu/dags)",
|
||||
)
|
||||
cmd.Flags().StringP("host", "s", "", "server host (default is localhost)")
|
||||
cmd.Flags().StringP("port", "p", "", "server port (default is 8080)")
|
||||
|
||||
@ -2,26 +2,28 @@ package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/test"
|
||||
)
|
||||
|
||||
func TestStartCommand(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
tests := []cmdTest{
|
||||
{
|
||||
args: []string{"start", testDAGFile("start.yaml")},
|
||||
args: []string{"start", testDAGFile("success.yaml")},
|
||||
expectedOut: []string{"1 finished"},
|
||||
},
|
||||
{
|
||||
args: []string{"start", testDAGFile("start_with_params.yaml")},
|
||||
args: []string{"start", testDAGFile("params.yaml")},
|
||||
expectedOut: []string{"params is p1 and p2"},
|
||||
},
|
||||
{
|
||||
args: []string{
|
||||
"start",
|
||||
`--params="p3 p4"`,
|
||||
testDAGFile("start_with_params.yaml"),
|
||||
testDAGFile("params.yaml"),
|
||||
},
|
||||
expectedOut: []string{"params is p3 and p4"},
|
||||
},
|
||||
|
||||
@ -4,14 +4,15 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/dag/scheduler"
|
||||
"github.com/dagu-dev/dagu/internal/test"
|
||||
)
|
||||
|
||||
func TestStatusCommand(t *testing.T) {
|
||||
t.Run("Status command should run", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("StatusDAG", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
dagFile := testDAGFile("status.yaml")
|
||||
dagFile := testDAGFile("long.yaml")
|
||||
|
||||
// Start the DAG.
|
||||
done := make(chan struct{})
|
||||
@ -20,7 +21,12 @@ func TestStatusCommand(t *testing.T) {
|
||||
close(done)
|
||||
}()
|
||||
|
||||
testLastStatusEventual(t, setup.dataStore.HistoryStore(), dagFile, scheduler.StatusRunning)
|
||||
testLastStatusEventual(
|
||||
t,
|
||||
setup.DataStore().HistoryStore(),
|
||||
dagFile,
|
||||
scheduler.StatusRunning,
|
||||
)
|
||||
|
||||
// Check the current status.
|
||||
testRunCommand(t, statusCmd(), cmdTest{
|
||||
|
||||
@ -5,14 +5,15 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/dag/scheduler"
|
||||
"github.com/dagu-dev/dagu/internal/test"
|
||||
)
|
||||
|
||||
func TestStopCommand(t *testing.T) {
|
||||
t.Run("Stop a DAG", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("StopDAG", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
dagFile := testDAGFile("stop.yaml")
|
||||
dagFile := testDAGFile("long2.yaml")
|
||||
|
||||
// Start the DAG.
|
||||
done := make(chan struct{})
|
||||
@ -24,7 +25,12 @@ func TestStopCommand(t *testing.T) {
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
|
||||
// Wait for the DAG running.
|
||||
testLastStatusEventual(t, setup.dataStore.HistoryStore(), dagFile, scheduler.StatusRunning)
|
||||
testLastStatusEventual(
|
||||
t,
|
||||
setup.DataStore().HistoryStore(),
|
||||
dagFile,
|
||||
scheduler.StatusRunning,
|
||||
)
|
||||
|
||||
// Stop the DAG.
|
||||
testRunCommand(t, stopCmd(), cmdTest{
|
||||
@ -32,7 +38,9 @@ func TestStopCommand(t *testing.T) {
|
||||
expectedOut: []string{"Stopping..."}})
|
||||
|
||||
// Check the last execution is cancelled.
|
||||
testLastStatusEventual(t, setup.dataStore.HistoryStore(), dagFile, scheduler.StatusCancel)
|
||||
testLastStatusEventual(
|
||||
t, setup.DataStore().HistoryStore(), dagFile, scheduler.StatusCancel,
|
||||
)
|
||||
<-done
|
||||
})
|
||||
}
|
||||
|
||||
3
cmd/testdata/long2.yaml
vendored
Normal file
3
cmd/testdata/long2.yaml
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
steps:
|
||||
- name: "1"
|
||||
command: "sleep 1000"
|
||||
3
cmd/testdata/status.yaml
vendored
3
cmd/testdata/status.yaml
vendored
@ -1,3 +0,0 @@
|
||||
steps:
|
||||
- name: "1"
|
||||
command: "sleep 1000"
|
||||
@ -1,41 +1,36 @@
|
||||
version: "3.9"
|
||||
services:
|
||||
|
||||
# init container updates permission
|
||||
init:
|
||||
image: "ghcr.io/dagu-dev/dagu:latest"
|
||||
user: root
|
||||
volumes:
|
||||
- dagu:/home/dagu/.dagu
|
||||
command: chown -R dagu /home/dagu/.dagu/
|
||||
|
||||
# ui web server process
|
||||
server:
|
||||
image: "ghcr.io/dagu-dev/dagu:latest"
|
||||
environment:
|
||||
- DAGU_PORT=8080
|
||||
- DAGU_DAGS=/home/dagu/.dagu/dags
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- dagu:/home/dagu/.dagu
|
||||
- ./dags/:/home/dagu/.dagu/dags
|
||||
depends_on:
|
||||
- init
|
||||
|
||||
# scheduler process
|
||||
scheduler:
|
||||
image: "ghcr.io/dagu-dev/dagu:latest"
|
||||
environment:
|
||||
- DAGU_DAGS=/home/dagu/.dagu/dags
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- dagu:/home/dagu/.dagu
|
||||
- ./dags/:/home/dagu/.dagu/dags
|
||||
command: dagu scheduler
|
||||
depends_on:
|
||||
- init
|
||||
|
||||
# init container updates permission
|
||||
init:
|
||||
image: "ghcr.io/dagu-dev/dagu:latest"
|
||||
user: root
|
||||
volumes:
|
||||
- dagu_config:/home/dagu/.config/dagu
|
||||
- dagu_data:/home/dagu/.local/share
|
||||
command: chown -R dagu /home/dagu/.config/dagu/ /home/dagu/.local/share
|
||||
# ui server process
|
||||
server:
|
||||
image: "ghcr.io/dagu-dev/dagu:latest"
|
||||
environment:
|
||||
- DAGU_PORT=8080
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- dagu_config:/home/dagu/.config/dagu
|
||||
- dagu_data:/home/dagu/.local/share
|
||||
depends_on:
|
||||
- init
|
||||
# scheduler process
|
||||
scheduler:
|
||||
image: "ghcr.io/dagu-dev/dagu:latest"
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- dagu_config:/home/dagu/.config/dagu
|
||||
- dagu_data:/home/dagu/.local/share
|
||||
command: dagu scheduler
|
||||
depends_on:
|
||||
- init
|
||||
volumes:
|
||||
dagu: {}
|
||||
dagu_config: {}
|
||||
dagu_data: {}
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
Base Configuration for all DAGs
|
||||
=====================================
|
||||
|
||||
Creating a base configuration (default path: ``~/.dagu/config.yaml``) is a convenient way to organize shared settings among all DAGs. The path to the base configuration file can be configured. See :ref:`Configuration Options` for more details.
|
||||
Creating a base configuration (default path: ``~/.config/dagu/base.yaml``) is a convenient way to organize shared settings among all DAGs. The path to the base configuration file can be configured. See :ref:`Configuration Options` for more details.
|
||||
|
||||
Example:
|
||||
|
||||
|
||||
@ -23,7 +23,7 @@ The following environment variables can be used to configure the Dagu. Default v
|
||||
- ``DAGU_DATA_DIR`` (``$DAGU_HOME/data``): The directory where application data will be stored.
|
||||
- ``DAGU_SUSPEND_FLAGS_DIR`` (``$DAGU_HOME/suspend``): The directory containing DAG suspend flags.
|
||||
- ``DAGU_ADMIN_LOG_DIR`` (``$DAGU_HOME/logs/admin``): The directory where admin logs will be stored.
|
||||
- ``DAGU_BASE_CONFIG`` (``$DAGU_HOME/config.yaml``): The path to the base configuration file.
|
||||
- ``DAGU_BASE_CONFIG`` (``$DAGU_HOME/base.yaml``): The path to the base configuration file.
|
||||
- ``DAGU_NAVBAR_COLOR`` (``""``): The color to use for the navigation bar. E.g., ``red`` or ``#ff0000``.
|
||||
- ``DAGU_NAVBAR_TITLE`` (``Dagu``): The title to display in the navigation bar. E.g., ``Dagu - PROD`` or ``Dagu - DEV``
|
||||
- ``DAGU_WORK_DIR``: The working directory for DAGs. If not set, the default value is DAG location. Also you can set the working directory for each DAG steps in the DAG configuration file. For more information, see :ref:`specifying working dir`.
|
||||
@ -62,7 +62,7 @@ You can create ``admin.yaml`` file in the ``$DAGU_HOME`` directory (default: ``$
|
||||
authToken: <token for API access> # API token
|
||||
|
||||
# Base Config
|
||||
baseConfig: <base DAG config path> # default: ${DAGU_HOME}/config.yaml
|
||||
baseConfig: <base DAG config path> # default: ${DAGU_HOME}/base.yaml
|
||||
|
||||
# Working Directory
|
||||
workDir: <working directory for DAGs> # default: DAG location
|
||||
|
||||
@ -27,13 +27,13 @@ msgstr "すべてのDAGの基本設定"
|
||||
#: ../../source/base_config.rst:6 0f42fc55990a404f9e436073ba71ba93
|
||||
#, fuzzy
|
||||
msgid ""
|
||||
"Creating a base configuration (default path: ``~/.dagu/config.yaml``) is "
|
||||
"Creating a base configuration (default path: ``~/.config/dagu/base.yaml``) is "
|
||||
"a convenient way to organize shared settings among all workflows. The "
|
||||
"path to the base configuration file can be configured. See "
|
||||
":ref:`Configuration Options` for more details."
|
||||
msgstr ""
|
||||
"基本設定(デフォルトのパス: "
|
||||
"``~/.dagu/config.yaml``)を作成することは、すべてのDAG間で共有設定を整理するのに便利な方法です。基本設定ファイルへのパスは設定可能です。詳細については、:ref:`Configuration"
|
||||
"``~/.config/dagu/base.yaml``)を作成することは、すべてのDAG間で共有設定を整理するのに便利な方法です。基本設定ファイルへのパスは設定可能です。詳細については、:ref:`Configuration"
|
||||
" Options` を参照してください。"
|
||||
|
||||
#: ../../source/base_config.rst:8 b7458496869246558d3efeee58648edd
|
||||
|
||||
@ -95,10 +95,10 @@ msgstr ""
|
||||
|
||||
#: ../../source/config.rst:26 5198113ba7f54794bbeb0077ec25bdc2
|
||||
msgid ""
|
||||
"``DAGU_BASE_CONFIG`` (``$DAGU_HOME/config.yaml``): The path to the base "
|
||||
"``DAGU_BASE_CONFIG`` (``$DAGU_HOME/base.yaml``): The path to the base "
|
||||
"configuration file."
|
||||
msgstr ""
|
||||
"``DAGU_BASE_CONFIG`` (``$DAGU_HOME/config.yaml``): 基本設定ファイルへのパス。"
|
||||
"``DAGU_BASE_CONFIG`` (``$DAGU_HOME/base.yaml``): 基本設定ファイルへのパス。"
|
||||
|
||||
#: ../../source/config.rst:27 f185a5aa23a049b484202296d52e9272
|
||||
msgid ""
|
||||
|
||||
@ -493,19 +493,19 @@ msgstr "``steps``: DAGで実行するステップのリスト。"
|
||||
|
||||
#: ../../source/yaml_format.rst:482 d2ef99deab3945e59788eee68e7d87f5
|
||||
msgid ""
|
||||
"In addition, a global configuration file, ``$DAGU_HOME/config.yaml``, can"
|
||||
"In addition, a global configuration file, ``$DAGU_HOME/base.yaml``, can"
|
||||
" be used to gather common settings, such as ``logDir`` or ``env``."
|
||||
msgstr ""
|
||||
"さらに、グローバル設定ファイル ``$DAGU_HOME/config.yaml`` を使用して、"
|
||||
"さらに、グローバル設定ファイル ``$DAGU_HOME/base.yaml`` を使用して、"
|
||||
"``logDir`` や ``env`` などの共通設定を集めることができます。"
|
||||
|
||||
#: ../../source/yaml_format.rst:484 9a4859890b2f49c5a66923630d9282e6
|
||||
msgid ""
|
||||
"Note: If ``DAGU_HOME`` environment variable is not set, the default path "
|
||||
"is ``$HOME/.dagu/config.yaml``."
|
||||
"is ``$HOME/.dagu/base.yaml``."
|
||||
msgstr ""
|
||||
"注意: ``DAGU_HOME`` 環境変数が設定されていない場合、デフォルトのパスは"
|
||||
" ``$HOME/.dagu/config.yaml`` です。"
|
||||
" ``$HOME/.dagu/base.yaml`` です。"
|
||||
|
||||
#: ../../source/yaml_format.rst:522 d8004407254d47db8fda9e238eadc212
|
||||
msgid "Step"
|
||||
|
||||
@ -26,12 +26,12 @@ msgstr "所有工作流的基本配置"
|
||||
#: ../../source/base_config.rst:6 0f42fc55990a404f9e436073ba71ba93
|
||||
#, fuzzy
|
||||
msgid ""
|
||||
"Creating a base configuration (default path: ``~/.dagu/config.yaml``) is "
|
||||
"Creating a base configuration (default path: ``~/.config/dagu/base.yaml``) is "
|
||||
"a convenient way to organize shared settings among all workflows. The "
|
||||
"path to the base configuration file can be configured. See "
|
||||
":ref:`Configuration Options` for more details."
|
||||
msgstr ""
|
||||
"创建基本配置(默认路径:``~/.dagu/config.yaml``)是组织所有工作流之间共享设置的一种便捷方式。可以配置基本配置文件的路径。有关更多详细信息,请参阅"
|
||||
"创建基本配置(默认路径:``~/.config/dagu/base.yaml``)是组织所有工作流之间共享设置的一种便捷方式。可以配置基本配置文件的路径。有关更多详细信息,请参阅"
|
||||
" :ref:`配置选项`。"
|
||||
|
||||
#: ../../source/base_config.rst:8 b7458496869246558d3efeee58648edd
|
||||
|
||||
@ -86,7 +86,7 @@ msgstr ""
|
||||
|
||||
#: ../../source/config.rst:26 2ab3dddd69504f17a31a31ec15cc5299
|
||||
msgid ""
|
||||
"``DAGU_BASE_CONFIG`` (``$DAGU_HOME/config.yaml``): The path to the base "
|
||||
"``DAGU_BASE_CONFIG`` (``$DAGU_HOME/base.yaml``): The path to the base "
|
||||
"configuration file."
|
||||
msgstr ""
|
||||
|
||||
|
||||
@ -477,17 +477,17 @@ msgstr "``steps``: 要在 DAG 中执行的步骤列表。"
|
||||
|
||||
#: ../../source/yaml_format.rst:482 8993958642c54df8af8f8feb63a61306
|
||||
msgid ""
|
||||
"In addition, a global configuration file, ``$DAGU_HOME/config.yaml``, can"
|
||||
"In addition, a global configuration file, ``$DAGU_HOME/base.yaml``, can"
|
||||
" be used to gather common settings, such as ``logDir`` or ``env``."
|
||||
msgstr ""
|
||||
"此外,可以使用全局配置文件 ``$DAGU_HOME/config.yaml`` 来收集常见设置,例如 ``logDir`` 或 ``env``。"
|
||||
"此外,可以使用全局配置文件 ``$DAGU_HOME/base.yaml`` 来收集常见设置,例如 ``logDir`` 或 ``env``。"
|
||||
|
||||
#: ../../source/yaml_format.rst:484 bee44595f6004cc0b34f4e2bbfdb1217
|
||||
msgid ""
|
||||
"Note: If ``DAGU_HOME`` environment variable is not set, the default path "
|
||||
"is ``$HOME/.dagu/config.yaml``."
|
||||
"is ``$HOME/.dagu/base.yaml``."
|
||||
msgstr ""
|
||||
"注意:如果未设置 ``DAGU_HOME`` 环境变量,默认路径为 ``$HOME/.dagu/config.yaml``。"
|
||||
"注意:如果未设置 ``DAGU_HOME`` 环境变量,默认路径为 ``$HOME/.dagu/base.yaml``。"
|
||||
|
||||
#: ../../source/yaml_format.rst:522 f7dc490d92784e0695de6deee2b84465
|
||||
msgid "Step"
|
||||
|
||||
@ -520,9 +520,9 @@ This section provides a comprehensive list of available fields that can be used
|
||||
- ``handlerOn``: The command to execute when a DAG or step succeeds, fails, cancels, or exits.
|
||||
- ``steps``: A list of steps to execute in the DAG.
|
||||
|
||||
In addition, a global configuration file, ``$DAGU_HOME/config.yaml``, can be used to gather common settings, such as ``logDir`` or ``env``.
|
||||
In addition, a global configuration file, ``$DAGU_HOME/base.yaml``, can be used to gather common settings, such as ``logDir`` or ``env``.
|
||||
|
||||
Note: If ``DAGU_HOME`` environment variable is not set, the default path is ``$HOME/.dagu/config.yaml``.
|
||||
Note: If ``DAGU_HOME`` environment variable is not set, the default path is ``$HOME/.dagu/base.yaml``.
|
||||
|
||||
Example:
|
||||
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
params: "`date +'%Y%m%d'`"
|
||||
schedule: "*/2 * * * *"
|
||||
steps:
|
||||
- name: step 1
|
||||
command: echo $1
|
||||
- name: step 2
|
||||
command: sleep 1
|
||||
depends:
|
||||
- step 1
|
||||
@ -1,41 +0,0 @@
|
||||
version: "3.9"
|
||||
services:
|
||||
|
||||
# init container updates permission
|
||||
init:
|
||||
image: "ghcr.io/dagu-dev/dagu:latest"
|
||||
user: root
|
||||
volumes:
|
||||
- dagu:/home/dagu/.dagu
|
||||
command: chown -R dagu /home/dagu/.dagu/
|
||||
|
||||
# ui server process
|
||||
server:
|
||||
image: "ghcr.io/dagu-dev/dagu:latest"
|
||||
environment:
|
||||
- DAGU_PORT=8080
|
||||
- DAGU_DAGS=/home/dagu/.dagu/dags
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- dagu:/home/dagu/.dagu
|
||||
- ./dags/:/home/dagu/.dagu/dags
|
||||
depends_on:
|
||||
- init
|
||||
|
||||
# scheduler process
|
||||
scheduler:
|
||||
image: "ghcr.io/dagu-dev/dagu:latest"
|
||||
environment:
|
||||
- DAGU_DAGS=/home/dagu/.dagu/dags
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- dagu:/home/dagu/.dagu
|
||||
- ./dags/:/home/dagu/.dagu/dags
|
||||
command: dagu scheduler
|
||||
depends_on:
|
||||
- init
|
||||
|
||||
volumes:
|
||||
dagu: {}
|
||||
6
go.mod
6
go.mod
@ -1,8 +1,9 @@
|
||||
module github.com/dagu-dev/dagu
|
||||
|
||||
go 1.22.0
|
||||
go 1.22.5
|
||||
|
||||
require (
|
||||
github.com/adrg/xdg v0.5.0
|
||||
github.com/docker/docker v20.10.21+incompatible
|
||||
github.com/fsnotify/fsnotify v1.7.0
|
||||
github.com/go-chi/chi/v5 v5.0.8
|
||||
@ -29,6 +30,7 @@ require (
|
||||
github.com/spf13/cobra v1.8.0
|
||||
github.com/spf13/viper v1.18.2
|
||||
github.com/stretchr/testify v1.9.0
|
||||
github.com/yohamta/gomerger v0.0.1
|
||||
go.uber.org/fx v1.20.0
|
||||
go.uber.org/goleak v1.3.0
|
||||
golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
|
||||
@ -261,6 +263,6 @@ require (
|
||||
github.com/samber/lo v1.38.1
|
||||
golang.org/x/crypto v0.24.0
|
||||
golang.org/x/net v0.26.0
|
||||
golang.org/x/sys v0.21.0
|
||||
golang.org/x/sys v0.22.0
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
8
go.sum
8
go.sum
@ -68,6 +68,8 @@ github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2y
|
||||
github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE=
|
||||
github.com/OpenPeeDeeP/depguard/v2 v2.2.0 h1:vDfG60vDtIuf0MEOhmLlLLSzqaRM8EMcgJPdp74zmpA=
|
||||
github.com/OpenPeeDeeP/depguard/v2 v2.2.0/go.mod h1:CIzddKRvLBC4Au5aYP/i3nyaWQ+ClszLIuVocRiCYFQ=
|
||||
github.com/adrg/xdg v0.5.0 h1:dDaZvhMXatArP1NPHhnfaQUqWBLBsmx1h1HXQdMoFCY=
|
||||
github.com/adrg/xdg v0.5.0/go.mod h1:dDdY4M4DF9Rjy4kHPeNL+ilVF+p2lK8IdM9/rTSGcI4=
|
||||
github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk=
|
||||
github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
|
||||
github.com/alecthomas/go-check-sumtype v0.1.4 h1:WCvlB3l5Vq5dZQTFmodqL2g68uHiSwwlWcT5a2FGK0c=
|
||||
@ -699,6 +701,8 @@ github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5Jsjqto
|
||||
github.com/yeya24/promlinter v0.3.0/go.mod h1:cDfJQQYv9uYciW60QT0eeHlFodotkYZlL+YcPQN+mW4=
|
||||
github.com/ykadowak/zerologlint v0.1.5 h1:Gy/fMz1dFQN9JZTPjv1hxEk+sRWm05row04Yoolgdiw=
|
||||
github.com/ykadowak/zerologlint v0.1.5/go.mod h1:KaUskqF3e/v59oPmdq1U1DnKcuHokl2/K1U4pmIELKg=
|
||||
github.com/yohamta/gomerger v0.0.1 h1:FFKDbEwp3iaFvAGqgkKQLnViX8J953hz7QYhII9h5Fw=
|
||||
github.com/yohamta/gomerger v0.0.1/go.mod h1:1ABCGxp1LkuJIeTJ2Ik0d2EOK2p9iRTesKWz5X1Tu+E=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
@ -929,8 +933,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
|
||||
@ -7,7 +7,6 @@ import (
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sync"
|
||||
@ -528,13 +527,13 @@ func (a *Agent) setupLog() error {
|
||||
// It is DAG.LogDir + DAG.Name (with invalid characters replaced with '_').
|
||||
// It is used to write the stdout and stderr of the steps.
|
||||
if a.dag.LogDir != "" {
|
||||
a.logDir = path.Join(a.logDir, util.ValidFilename(a.dag.Name))
|
||||
a.logDir = filepath.Join(a.logDir, util.ValidFilename(a.dag.Name))
|
||||
}
|
||||
|
||||
absFilepath := filepath.Join(a.logDir, createLogfileName(a.dag.Name, a.reqID, time.Now()))
|
||||
|
||||
// Create the log directory
|
||||
if err := os.MkdirAll(path.Dir(absFilepath), 0755); err != nil {
|
||||
if err := os.MkdirAll(filepath.Dir(absFilepath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@ -4,78 +4,33 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/agent"
|
||||
"github.com/dagu-dev/dagu/internal/persistence"
|
||||
"github.com/dagu-dev/dagu/internal/persistence/client"
|
||||
"github.com/dagu-dev/dagu/internal/test"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/config"
|
||||
"github.com/dagu-dev/dagu/internal/dag"
|
||||
"github.com/dagu-dev/dagu/internal/dag/scheduler"
|
||||
"github.com/dagu-dev/dagu/internal/engine"
|
||||
"github.com/dagu-dev/dagu/internal/persistence/model"
|
||||
"github.com/dagu-dev/dagu/internal/util"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type testSetup struct {
|
||||
homeDir string
|
||||
engine engine.Engine
|
||||
dataStore persistence.DataStores
|
||||
cfg *config.Config
|
||||
}
|
||||
|
||||
func (t testSetup) cleanup() {
|
||||
_ = os.RemoveAll(t.homeDir)
|
||||
}
|
||||
|
||||
// setupTest sets temporary directories and loads the configuration.
|
||||
func setupTest(t *testing.T) testSetup {
|
||||
t.Helper()
|
||||
|
||||
tmpDir := util.MustTempDir("dagu_test")
|
||||
err := os.Setenv("HOME", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg, err := config.Load()
|
||||
require.NoError(t, err)
|
||||
|
||||
dataStore := client.NewDataStores(&client.NewDataStoresArgs{
|
||||
DataDir: path.Join(tmpDir, ".dagu", "data"),
|
||||
})
|
||||
|
||||
eng := engine.New(&engine.NewEngineArgs{
|
||||
DataStore: dataStore,
|
||||
WorkDir: cfg.WorkDir,
|
||||
Executable: cfg.Executable,
|
||||
})
|
||||
|
||||
return testSetup{
|
||||
homeDir: tmpDir,
|
||||
engine: eng,
|
||||
dataStore: dataStore,
|
||||
cfg: cfg,
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgent_Run(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("Run a DAG successfully", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("RunDAG", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
dg := testLoadDAG(t, "run.yaml")
|
||||
eng := setup.engine
|
||||
eng := setup.Engine()
|
||||
dagAgent := agent.New(&agent.NewAagentArgs{
|
||||
DAG: dg,
|
||||
LogDir: setup.cfg.LogDir,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: eng,
|
||||
DataStore: setup.dataStore,
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
|
||||
latestStatus, err := eng.GetLatestStatus(dg)
|
||||
@ -95,18 +50,18 @@ func TestAgent_Run(t *testing.T) {
|
||||
return status.Status == scheduler.StatusSuccess
|
||||
}, time.Second*2, time.Millisecond*100)
|
||||
})
|
||||
t.Run("Old history files are deleted", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("DeleteOldHistory", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
// Create a history file by running a DAG
|
||||
dg := testLoadDAG(t, "simple.yaml")
|
||||
eng := setup.engine
|
||||
eng := setup.Engine()
|
||||
dagAgent := agent.New(&agent.NewAagentArgs{
|
||||
DAG: dg,
|
||||
LogDir: setup.cfg.LogDir,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: eng,
|
||||
DataStore: setup.dataStore,
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
err := dagAgent.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
@ -117,9 +72,9 @@ func TestAgent_Run(t *testing.T) {
|
||||
dg.HistRetentionDays = 0
|
||||
dagAgent = agent.New(&agent.NewAagentArgs{
|
||||
DAG: dg,
|
||||
LogDir: setup.cfg.LogDir,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: eng,
|
||||
DataStore: setup.dataStore,
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
err = dagAgent.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
@ -128,17 +83,17 @@ func TestAgent_Run(t *testing.T) {
|
||||
history = eng.GetRecentHistory(dg, 2)
|
||||
require.Equal(t, 1, len(history))
|
||||
})
|
||||
t.Run("It should not run a DAG if it is already running", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("AlreadyRunning", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
dg := testLoadDAG(t, "is_running.yaml")
|
||||
eng := setup.engine
|
||||
eng := setup.Engine()
|
||||
dagAgent := agent.New(&agent.NewAagentArgs{
|
||||
DAG: dg,
|
||||
LogDir: setup.cfg.LogDir,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: eng,
|
||||
DataStore: setup.dataStore,
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
|
||||
go func() {
|
||||
@ -153,29 +108,29 @@ func TestAgent_Run(t *testing.T) {
|
||||
|
||||
dagAgent = agent.New(&agent.NewAagentArgs{
|
||||
DAG: dg,
|
||||
LogDir: setup.cfg.LogDir,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: eng,
|
||||
DataStore: setup.dataStore,
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
err := dagAgent.Run(context.Background())
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "is already running")
|
||||
})
|
||||
t.Run("It should not run a DAG if the precondition is not met", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("PreConditionNotMet", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
dg := testLoadDAG(t, "multiple_steps.yaml")
|
||||
eng := setup.engine
|
||||
eng := setup.Engine()
|
||||
|
||||
// Precondition is not met
|
||||
dg.Preconditions = []dag.Condition{{Condition: "`echo 1`", Expected: "0"}}
|
||||
|
||||
dagAgent := agent.New(&agent.NewAagentArgs{
|
||||
DAG: dg,
|
||||
LogDir: setup.cfg.LogDir,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: eng,
|
||||
DataStore: setup.dataStore,
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
err := dagAgent.Run(context.Background())
|
||||
require.Error(t, err)
|
||||
@ -186,16 +141,16 @@ func TestAgent_Run(t *testing.T) {
|
||||
require.Equal(t, scheduler.NodeStatusNone, status.Nodes[0].Status)
|
||||
require.Equal(t, scheduler.NodeStatusNone, status.Nodes[1].Status)
|
||||
})
|
||||
t.Run("Run a DAG and finish with an error", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("FinishWithError", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
// Run a DAG that fails
|
||||
dagAgent := agent.New(&agent.NewAagentArgs{
|
||||
DAG: testLoadDAG(t, "error.yaml"),
|
||||
LogDir: setup.cfg.LogDir,
|
||||
Engine: setup.engine,
|
||||
DataStore: setup.dataStore,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: setup.Engine(),
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
err := dagAgent.Run(context.Background())
|
||||
require.Error(t, err)
|
||||
@ -203,19 +158,19 @@ func TestAgent_Run(t *testing.T) {
|
||||
// Check if the status is saved correctly
|
||||
require.Equal(t, scheduler.StatusError, dagAgent.Status().Status)
|
||||
})
|
||||
t.Run("Run a DAG and receive a signal", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("ReceiveSignal", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
abortFunc := func(a *agent.Agent) { a.Signal(syscall.SIGTERM) }
|
||||
|
||||
dg := testLoadDAG(t, "sleep.yaml")
|
||||
eng := setup.engine
|
||||
eng := setup.Engine()
|
||||
dagAgent := agent.New(&agent.NewAagentArgs{
|
||||
DAG: dg,
|
||||
LogDir: setup.cfg.LogDir,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: eng,
|
||||
DataStore: setup.dataStore,
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
|
||||
go func() {
|
||||
@ -238,16 +193,16 @@ func TestAgent_Run(t *testing.T) {
|
||||
return status.Status == scheduler.StatusCancel
|
||||
}, time.Second*1, time.Millisecond*100)
|
||||
})
|
||||
t.Run("Run a DAG and execute the exit handler", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("ExitHandler", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
dg := testLoadDAG(t, "on_exit.yaml")
|
||||
dagAgent := agent.New(&agent.NewAagentArgs{
|
||||
DAG: dg,
|
||||
LogDir: setup.cfg.LogDir,
|
||||
Engine: setup.engine,
|
||||
DataStore: setup.dataStore,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: setup.Engine(),
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
err := dagAgent.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
@ -266,18 +221,18 @@ func TestAgent_Run(t *testing.T) {
|
||||
|
||||
func TestAgent_DryRun(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("Dry-run a DAG successfully", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("DryRun", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
dg := testLoadDAG(t, "dry.yaml")
|
||||
eng := setup.engine
|
||||
eng := setup.Engine()
|
||||
dagAgent := agent.New(&agent.NewAagentArgs{
|
||||
DAG: dg,
|
||||
Dry: true,
|
||||
LogDir: setup.cfg.LogDir,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: eng,
|
||||
DataStore: setup.dataStore,
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
|
||||
err := dagAgent.Run(context.Background())
|
||||
@ -295,19 +250,19 @@ func TestAgent_DryRun(t *testing.T) {
|
||||
|
||||
func TestAgent_Retry(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("Retry a DAG", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("RetryDAG", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
// retry.yaml has a DAG that fails
|
||||
dg := testLoadDAG(t, "retry.yaml")
|
||||
eng := setup.engine
|
||||
eng := setup.Engine()
|
||||
|
||||
dagAgent := agent.New(&agent.NewAagentArgs{
|
||||
DAG: dg,
|
||||
LogDir: setup.cfg.LogDir,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: eng,
|
||||
DataStore: setup.dataStore,
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
err := dagAgent.Run(context.Background())
|
||||
require.Error(t, err)
|
||||
@ -325,9 +280,9 @@ func TestAgent_Retry(t *testing.T) {
|
||||
dagAgent = agent.New(&agent.NewAagentArgs{
|
||||
DAG: dg,
|
||||
RetryTarget: status,
|
||||
LogDir: setup.cfg.LogDir,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: eng,
|
||||
DataStore: setup.dataStore,
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
err = dagAgent.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
@ -346,18 +301,18 @@ func TestAgent_Retry(t *testing.T) {
|
||||
|
||||
func TestAgent_HandleHTTP(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("Handle HTTP requests and return the status of the DAG", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("HTTP_Valid", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
// Start a long-running DAG
|
||||
dg := testLoadDAG(t, "handle_http.yaml")
|
||||
eng := setup.engine
|
||||
eng := setup.Engine()
|
||||
dagAgent := agent.New(&agent.NewAagentArgs{
|
||||
DAG: dg,
|
||||
LogDir: setup.cfg.LogDir,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: eng,
|
||||
DataStore: setup.dataStore,
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
go func() {
|
||||
err := dagAgent.Run(context.Background())
|
||||
@ -392,18 +347,18 @@ func TestAgent_HandleHTTP(t *testing.T) {
|
||||
}, time.Second*2, time.Millisecond*100)
|
||||
|
||||
})
|
||||
t.Run("Handle invalid HTTP requests", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("HTTP_InvalidRequest", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
// Start a long-running DAG
|
||||
dg := testLoadDAG(t, "handle_http2.yaml")
|
||||
eng := setup.engine
|
||||
eng := setup.Engine()
|
||||
dagAgent := agent.New(&agent.NewAagentArgs{
|
||||
DAG: dg,
|
||||
LogDir: setup.cfg.LogDir,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: eng,
|
||||
DataStore: setup.dataStore,
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
|
||||
go func() {
|
||||
@ -435,18 +390,18 @@ func TestAgent_HandleHTTP(t *testing.T) {
|
||||
return status.Status == scheduler.StatusCancel
|
||||
}, time.Second*2, time.Millisecond*100)
|
||||
})
|
||||
t.Run("Handle cancel request and stop the DAG", func(t *testing.T) {
|
||||
setup := setupTest(t)
|
||||
defer setup.cleanup()
|
||||
t.Run("HTTP_HandleCancel", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
// Start a long-running DAG
|
||||
dg := testLoadDAG(t, "handle_http3.yaml")
|
||||
eng := setup.engine
|
||||
eng := setup.Engine()
|
||||
dagAgent := agent.New(&agent.NewAagentArgs{
|
||||
DAG: dg,
|
||||
LogDir: setup.cfg.LogDir,
|
||||
LogDir: setup.Config.LogDir,
|
||||
Engine: eng,
|
||||
DataStore: setup.dataStore,
|
||||
DataStore: setup.DataStore(),
|
||||
})
|
||||
|
||||
go func() {
|
||||
@ -507,7 +462,7 @@ func (h *mockResponseWriter) WriteHeader(statusCode int) {
|
||||
// testLoadDAG load the specified DAG file for testing
|
||||
// without base config or parameters.
|
||||
func testLoadDAG(t *testing.T, name string) *dag.DAG {
|
||||
file := path.Join(util.MustGetwd(), "testdata", name)
|
||||
file := filepath.Join(util.MustGetwd(), "testdata", name)
|
||||
dg, err := dag.Load("", file, "")
|
||||
require.NoError(t, err)
|
||||
return dg
|
||||
|
||||
@ -5,7 +5,7 @@ import (
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/util"
|
||||
@ -27,7 +27,7 @@ func TestTeeWriter(t *testing.T) {
|
||||
}()
|
||||
|
||||
// Create a temporary file and tee the log to the file.
|
||||
tmpLogFile := path.Join(util.MustTempDir("test-tee"), "test.log")
|
||||
tmpLogFile := filepath.Join(util.MustTempDir("test-tee"), "test.log")
|
||||
logFile, err := os.Create(tmpLogFile)
|
||||
require.NoError(t, err)
|
||||
|
||||
|
||||
@ -2,15 +2,18 @@ package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/adrg/xdg"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// Config represents the configuration for the server.
|
||||
type Config struct {
|
||||
Host string // Server host
|
||||
Port int // Server port
|
||||
@ -41,22 +44,19 @@ type TLS struct {
|
||||
KeyFile string
|
||||
}
|
||||
|
||||
var lock sync.Mutex
|
||||
var configLock sync.Mutex
|
||||
|
||||
const envPrefix = "DAGU"
|
||||
|
||||
func Load() (*Config, error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
configLock.Lock()
|
||||
defer configLock.Unlock()
|
||||
|
||||
viper.SetEnvPrefix(envPrefix)
|
||||
viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
|
||||
|
||||
// Bind environment variables with config keys.
|
||||
bindEnvs()
|
||||
|
||||
// Set default values for config keys.
|
||||
if err := setDefaults(); err != nil {
|
||||
if err := setupViper(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -79,138 +79,226 @@ func Load() (*Config, error) {
|
||||
|
||||
// Set environment variables specified in the config file.
|
||||
cfg.Env.Range(func(k, v any) bool {
|
||||
_ = os.Setenv(k.(string), v.(string))
|
||||
if err := os.Setenv(k.(string), v.(string)); err != nil {
|
||||
log.Printf("failed to set env variable %s: %v", k, err)
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
var defaults = Config{
|
||||
Host: "127.0.0.1",
|
||||
Port: 8080,
|
||||
IsBasicAuth: false,
|
||||
NavbarTitle: "Dagu",
|
||||
IsAuthToken: false,
|
||||
LatestStatusToday: false,
|
||||
APIBaseURL: "/api/v1",
|
||||
AdminLogsDir: path.Join(logDir, "admin"),
|
||||
const (
|
||||
// Application name.
|
||||
appName = "dagu"
|
||||
)
|
||||
|
||||
func setupViper() error {
|
||||
// Bind environment variables with config keys.
|
||||
bindEnvs()
|
||||
|
||||
// Set default values for config keys.
|
||||
|
||||
// Directories
|
||||
baseDirs := getBaseDirs()
|
||||
viper.SetDefault("dags", baseDirs.dags)
|
||||
viper.SetDefault("suspendFlagsDir", baseDirs.suspendFlags)
|
||||
viper.SetDefault("dataDir", baseDirs.data)
|
||||
viper.SetDefault("logDir", baseDirs.logs)
|
||||
viper.SetDefault("adminLogsDir", baseDirs.adminLogs)
|
||||
|
||||
// Base config file
|
||||
viper.SetDefault("baseConfig", getBaseConfigPath(baseDirs))
|
||||
|
||||
// Other defaults
|
||||
viper.SetDefault("host", "127.0.0.1")
|
||||
viper.SetDefault("port", "8080")
|
||||
viper.SetDefault("navbarTitle", "Dagu")
|
||||
viper.SetDefault("apiBaseURL", "/api/v1")
|
||||
|
||||
// Set executable path
|
||||
// This is used for invoking the workflow process on the server.
|
||||
return setExecutableDefault()
|
||||
}
|
||||
|
||||
type baseDirs struct {
|
||||
config string
|
||||
dags string
|
||||
suspendFlags string
|
||||
data string
|
||||
logs string
|
||||
adminLogs string
|
||||
}
|
||||
|
||||
const (
|
||||
// Constants for config.
|
||||
appHomeDefault = ".dagu"
|
||||
legacyAppHome = "DAGU_HOME"
|
||||
|
||||
// Default base config file.
|
||||
baseConfig = "config.yaml"
|
||||
legacyConfigDir = ".dagu"
|
||||
legacyConfigDirEnvKey = "DAGU_HOME"
|
||||
|
||||
// default directories
|
||||
dagsDir = "dags"
|
||||
dataDir = "data"
|
||||
logDir = "logs"
|
||||
suspendDir = "suspend"
|
||||
)
|
||||
|
||||
func setDefaults() error {
|
||||
var (
|
||||
// ConfigDir is the directory to store DAGs and other configuration files.
|
||||
ConfigDir = getConfigDir()
|
||||
// DataDir is the directory to store history data.
|
||||
DataDir = getDataDir()
|
||||
// LogsDir is the directory to store logs.
|
||||
LogsDir = getLogsDir()
|
||||
)
|
||||
|
||||
func getBaseDirs() baseDirs {
|
||||
return baseDirs{
|
||||
config: ConfigDir,
|
||||
dags: filepath.Join(ConfigDir, dagsDir),
|
||||
suspendFlags: filepath.Join(ConfigDir, suspendDir),
|
||||
data: DataDir,
|
||||
logs: LogsDir,
|
||||
adminLogs: filepath.Join(LogsDir, "admin"),
|
||||
}
|
||||
}
|
||||
|
||||
func setExecutableDefault() error {
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get executable path: %w", err)
|
||||
}
|
||||
|
||||
paths := getDefaultPaths()
|
||||
|
||||
viper.SetDefault("host", defaults.Host)
|
||||
viper.SetDefault("port", defaults.Port)
|
||||
viper.SetDefault("executable", executable)
|
||||
viper.SetDefault("dags", path.Join(paths.configDir, dagsDir))
|
||||
viper.SetDefault("workDir", defaults.WorkDir)
|
||||
viper.SetDefault("isBasicAuth", defaults.IsBasicAuth)
|
||||
viper.SetDefault("basicAuthUsername", defaults.BasicAuthUsername)
|
||||
viper.SetDefault("basicAuthPassword", defaults.BasicAuthPassword)
|
||||
viper.SetDefault("logEncodingCharset", defaults.LogEncodingCharset)
|
||||
viper.SetDefault("baseConfig", path.Join(paths.configDir, baseConfig))
|
||||
viper.SetDefault("logDir", path.Join(paths.configDir, logDir))
|
||||
viper.SetDefault("dataDir", path.Join(paths.configDir, dataDir))
|
||||
viper.SetDefault("suspendFlagsDir", path.Join(paths.configDir, suspendDir))
|
||||
viper.SetDefault("adminLogsDir", path.Join(paths.configDir, defaults.AdminLogsDir))
|
||||
viper.SetDefault("navbarColor", defaults.NavbarColor)
|
||||
viper.SetDefault("navbarTitle", defaults.NavbarTitle)
|
||||
viper.SetDefault("isAuthToken", defaults.IsAuthToken)
|
||||
viper.SetDefault("authToken", defaults.AuthToken)
|
||||
viper.SetDefault("latestStatusToday", defaults.LatestStatusToday)
|
||||
viper.SetDefault("apiBaseURL", defaults.APIBaseURL)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getLogsDir() string {
|
||||
if v, ok := getLegacyConfigPath(); ok {
|
||||
// For backward compatibility.
|
||||
return filepath.Join(v, "logs")
|
||||
}
|
||||
return filepath.Join(xdg.DataHome, appName, "logs")
|
||||
}
|
||||
|
||||
func getDataDir() string {
|
||||
if v, ok := getLegacyConfigPath(); ok {
|
||||
// For backward compatibility.
|
||||
return filepath.Join(v, "data")
|
||||
}
|
||||
return filepath.Join(xdg.DataHome, appName, "history")
|
||||
}
|
||||
|
||||
func getConfigDir() string {
|
||||
if v, ok := getLegacyConfigPath(); ok {
|
||||
return v
|
||||
}
|
||||
if v := os.Getenv("XDG_CONFIG_HOME"); v != "" {
|
||||
return filepath.Join(v, appName)
|
||||
}
|
||||
return filepath.Join(getHomeDir(), ".config", appName)
|
||||
}
|
||||
|
||||
func getHomeDir() string {
|
||||
dir, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
log.Fatalf("could not determine home directory: %v", err)
|
||||
return ""
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
const (
|
||||
// Base config file name for all DAGs.
|
||||
baseConfig = "base.yaml"
|
||||
// Legacy config path for backward compatibility.
|
||||
legacyBaseConfig = "base.yaml"
|
||||
)
|
||||
|
||||
func getBaseConfigPath(b baseDirs) string {
|
||||
legacyPath := filepath.Join(b.config, legacyBaseConfig)
|
||||
if _, err := os.Stat(legacyPath); err == nil {
|
||||
return legacyPath
|
||||
}
|
||||
return filepath.Join(b.config, baseConfig)
|
||||
}
|
||||
|
||||
func getLegacyConfigPath() (string, bool) {
|
||||
// For backward compatibility.
|
||||
// If the environment variable is set, use it.
|
||||
if v := os.Getenv(legacyConfigDirEnvKey); v != "" {
|
||||
return v, true
|
||||
}
|
||||
// If not, check if the legacyPath config directory exists.
|
||||
legacyPath := filepath.Join(getHomeDir(), legacyConfigDir)
|
||||
if _, err := os.Stat(legacyPath); err == nil {
|
||||
return legacyPath, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func bindEnvs() {
|
||||
_ = viper.BindEnv("executable", "DAGU_EXECUTABLE")
|
||||
_ = viper.BindEnv("dags", "DAGU_DAGS_DIR")
|
||||
_ = viper.BindEnv("workDir", "DAGU_WORK_DIR")
|
||||
// Server configurations
|
||||
_ = viper.BindEnv("logEncodingCharset", "DAGU_LOG_ENCODING_CHARSET")
|
||||
_ = viper.BindEnv("navbarColor", "DAGU_NAVBAR_COLOR")
|
||||
_ = viper.BindEnv("navbarTitle", "DAGU_NAVBAR_TITLE")
|
||||
_ = viper.BindEnv("apiBaseURL", "DAGU_API_BASE_URL")
|
||||
|
||||
// Basic authentication
|
||||
_ = viper.BindEnv("isBasicAuth", "DAGU_IS_BASICAUTH")
|
||||
_ = viper.BindEnv("basicAuthUsername", "DAGU_BASICAUTH_USERNAME")
|
||||
_ = viper.BindEnv("basicAuthPassword", "DAGU_BASICAUTH_PASSWORD")
|
||||
_ = viper.BindEnv("logEncodingCharset", "DAGU_LOG_ENCODING_CHARSET")
|
||||
|
||||
// TLS configurations
|
||||
_ = viper.BindEnv("tls.certFile", "DAGU_CERT_FILE")
|
||||
_ = viper.BindEnv("tls.keyFile", "DAGU_KEY_FILE")
|
||||
|
||||
// Auth Token
|
||||
_ = viper.BindEnv("isAuthToken", "DAGU_IS_AUTHTOKEN")
|
||||
_ = viper.BindEnv("authToken", "DAGU_AUTHTOKEN")
|
||||
|
||||
// Executables
|
||||
_ = viper.BindEnv("executable", "DAGU_EXECUTABLE")
|
||||
|
||||
// Directories and files
|
||||
_ = viper.BindEnv("dags", "DAGU_DAGS_DIR")
|
||||
_ = viper.BindEnv("workDir", "DAGU_WORK_DIR")
|
||||
_ = viper.BindEnv("baseConfig", "DAGU_BASE_CONFIG")
|
||||
_ = viper.BindEnv("logDir", "DAGU_LOG_DIR")
|
||||
_ = viper.BindEnv("dataDir", "DAGU_DATA_DIR")
|
||||
_ = viper.BindEnv("suspendFlagsDir", "DAGU_SUSPEND_FLAGS_DIR")
|
||||
_ = viper.BindEnv("adminLogsDir", "DAGU_ADMIN_LOG_DIR")
|
||||
_ = viper.BindEnv("navbarColor", "DAGU_NAVBAR_COLOR")
|
||||
_ = viper.BindEnv("navbarTitle", "DAGU_NAVBAR_TITLE")
|
||||
_ = viper.BindEnv("tls.certFile", "DAGU_CERT_FILE")
|
||||
_ = viper.BindEnv("tls.keyFile", "DAGU_KEY_FILE")
|
||||
_ = viper.BindEnv("isAuthToken", "DAGU_IS_AUTHTOKEN")
|
||||
_ = viper.BindEnv("authToken", "DAGU_AUTHTOKEN")
|
||||
|
||||
// Miscellaneous
|
||||
_ = viper.BindEnv("latestStatusToday", "DAGU_LATEST_STATUS")
|
||||
_ = viper.BindEnv("apiBaseURL", "DAGU_API_BASE_URL")
|
||||
}
|
||||
|
||||
func loadLegacyEnvs(cfg *Config) {
|
||||
// For backward compatibility.
|
||||
// Load old environment variables if they exist.
|
||||
if v := os.Getenv("DAGU__ADMIN_NAVBAR_COLOR"); v != "" {
|
||||
log.Println("DAGU__ADMIN_NAVBAR_COLOR is deprecated. Use DAGU_NAVBAR_COLOR instead.")
|
||||
cfg.NavbarColor = v
|
||||
}
|
||||
if v := os.Getenv("DAGU__ADMIN_NAVBAR_TITLE"); v != "" {
|
||||
log.Println("DAGU__ADMIN_NAVBAR_TITLE is deprecated. Use DAGU_NAVBAR_TITLE instead.")
|
||||
cfg.NavbarTitle = v
|
||||
}
|
||||
if v := os.Getenv("DAGU__ADMIN_PORT"); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil {
|
||||
log.Println("DAGU__ADMIN_PORT is deprecated. Use DAGU_PORT instead.")
|
||||
cfg.Port = i
|
||||
}
|
||||
}
|
||||
if v := os.Getenv("DAGU__ADMIN_HOST"); v != "" {
|
||||
log.Println("DAGU__ADMIN_HOST is deprecated. Use DAGU_HOST instead.")
|
||||
cfg.Host = v
|
||||
}
|
||||
if v := os.Getenv("DAGU__DATA"); v != "" {
|
||||
log.Println("DAGU__DATA is deprecated. Use DAGU_DATA_DIR instead.")
|
||||
cfg.DataDir = v
|
||||
}
|
||||
if v := os.Getenv("DAGU__SUSPEND_FLAGS_DIR"); v != "" {
|
||||
log.Println("DAGU__SUSPEND_FLAGS_DIR is deprecated. Use DAGU_SUSPEND_FLAGS_DIR instead.")
|
||||
cfg.SuspendFlagsDir = v
|
||||
}
|
||||
if v := os.Getenv("DAGU__ADMIN_LOGS_DIR"); v != "" {
|
||||
log.Println("DAGU__ADMIN_LOGS_DIR is deprecated. Use DAGU_ADMIN_LOG_DIR instead.")
|
||||
cfg.AdminLogsDir = v
|
||||
}
|
||||
}
|
||||
|
||||
type defaultPaths struct {
|
||||
configDir string
|
||||
// Add more paths here if needed.
|
||||
}
|
||||
|
||||
func getDefaultPaths() defaultPaths {
|
||||
var paths defaultPaths
|
||||
|
||||
if appDir := os.Getenv(legacyAppHome); appDir != "" {
|
||||
paths.configDir = appDir
|
||||
} else {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
paths.configDir = path.Join(home, appHomeDefault)
|
||||
}
|
||||
|
||||
return paths
|
||||
}
|
||||
|
||||
@ -3,11 +3,10 @@ package dag
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/config"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@ -403,25 +402,23 @@ schedule:
|
||||
func TestLoad(t *testing.T) {
|
||||
// Base config has the following values:
|
||||
// MailOn: {Failure: true, Success: false}
|
||||
t.Run("WithBaseConfig", func(t *testing.T) {
|
||||
cfg, err := config.Load()
|
||||
require.NoError(t, err)
|
||||
t.Run("OverrideBaseConfig", func(t *testing.T) {
|
||||
baseConfig := filepath.Join(testdataDir, "base.yaml")
|
||||
|
||||
// Overwrite the base config with the following values:
|
||||
// MailOn: {Failure: false, Success: false}
|
||||
dg, err := Load(cfg.BaseConfig, path.Join(testdataDir, "overwrite.yaml"), "")
|
||||
dg, err := Load(baseConfig, filepath.Join(testdataDir, "overwrite.yaml"), "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// The MailOn key should be overwritten.
|
||||
require.Equal(t, &MailOn{Failure: false, Success: false}, dg.MailOn)
|
||||
require.Equal(t, dg.HistRetentionDays, 7)
|
||||
})
|
||||
t.Run("WithoutBaseConfig", func(t *testing.T) {
|
||||
cfg, err := config.Load()
|
||||
require.NoError(t, err)
|
||||
t.Run("NoOverrideBaseConfig", func(t *testing.T) {
|
||||
baseConfig := filepath.Join(testdataDir, "base.yaml")
|
||||
|
||||
// no_overwrite.yaml does not have the MailOn key.
|
||||
dg, err := Load(cfg.BaseConfig, path.Join(testdataDir, "no_overwrite.yaml"), "")
|
||||
dg, err := Load(baseConfig, filepath.Join(testdataDir, "no_overwrite.yaml"), "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// The MailOn key should be the same as the base config.
|
||||
|
||||
@ -4,7 +4,7 @@ import (
|
||||
// nolint // gosec
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@ -177,7 +177,7 @@ func (d *DAG) setup() {
|
||||
}
|
||||
|
||||
// set the default working directory for the steps if not set
|
||||
dir := path.Dir(d.Location)
|
||||
dir := filepath.Dir(d.Location)
|
||||
for i := range d.Steps {
|
||||
d.Steps[i].setup(dir)
|
||||
}
|
||||
@ -214,7 +214,7 @@ func (d *DAG) HasTag(tag string) bool {
|
||||
// run in parallel.
|
||||
func (d *DAG) SockAddr() string {
|
||||
s := strings.ReplaceAll(d.Location, " ", "_")
|
||||
name := strings.Replace(path.Base(s), path.Ext(path.Base(s)), "", 1)
|
||||
name := strings.Replace(filepath.Base(s), filepath.Ext(filepath.Base(s)), "", 1)
|
||||
// nolint // gosec
|
||||
h := md5.New()
|
||||
_, _ = h.Write([]byte(s))
|
||||
@ -226,7 +226,7 @@ func (d *DAG) SockAddr() string {
|
||||
if len(name) > lengthLimit {
|
||||
name = name[:lengthLimit-1]
|
||||
}
|
||||
return path.Join("/tmp", fmt.Sprintf("@dagu-%s-%x.sock", name, bs))
|
||||
return filepath.Join("/tmp", fmt.Sprintf("@dagu-%s-%x.sock", name, bs))
|
||||
}
|
||||
|
||||
// String implements the Stringer interface.
|
||||
|
||||
@ -1,8 +1,7 @@
|
||||
package dag
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/util"
|
||||
@ -10,21 +9,12 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
testdataDir = path.Join(util.MustGetwd(), "testdata")
|
||||
testHomeDir = path.Join(util.MustGetwd(), "testdata/home")
|
||||
testdataDir = filepath.Join(util.MustGetwd(), "testdata")
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
err := os.Setenv("HOME", testHomeDir)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func TestDAG_String(t *testing.T) {
|
||||
t.Run("DefaltConfig", func(t *testing.T) {
|
||||
dg, err := Load("", path.Join(testdataDir, "default.yaml"), "")
|
||||
dg, err := Load("", filepath.Join(testdataDir, "default.yaml"), "")
|
||||
require.NoError(t, err)
|
||||
|
||||
ret := dg.String()
|
||||
|
||||
@ -1,11 +1,10 @@
|
||||
package dag
|
||||
|
||||
import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/config"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@ -19,27 +18,27 @@ func Test_Load(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
name: "WithExt",
|
||||
file: path.Join(testdataDir, "loader_test.yaml"),
|
||||
expectedLocation: path.Join(testdataDir, "loader_test.yaml"),
|
||||
file: filepath.Join(testdataDir, "loader_test.yaml"),
|
||||
expectedLocation: filepath.Join(testdataDir, "loader_test.yaml"),
|
||||
},
|
||||
{
|
||||
name: "WithoutExt",
|
||||
file: path.Join(testdataDir, "loader_test"),
|
||||
expectedLocation: path.Join(testdataDir, "loader_test.yaml"),
|
||||
file: filepath.Join(testdataDir, "loader_test"),
|
||||
expectedLocation: filepath.Join(testdataDir, "loader_test.yaml"),
|
||||
},
|
||||
{
|
||||
name: "InvalidPath",
|
||||
file: path.Join(testdataDir, "not_existing_file.yaml"),
|
||||
file: filepath.Join(testdataDir, "not_existing_file.yaml"),
|
||||
expectedError: "no such file or directory",
|
||||
},
|
||||
{
|
||||
name: "InvalidDAG",
|
||||
file: path.Join(testdataDir, "err_decode.yaml"),
|
||||
file: filepath.Join(testdataDir, "err_decode.yaml"),
|
||||
expectedError: "has invalid keys: invalidkey",
|
||||
},
|
||||
{
|
||||
name: "InvalidYAML",
|
||||
file: path.Join(testdataDir, "err_parse.yaml"),
|
||||
file: filepath.Join(testdataDir, "err_parse.yaml"),
|
||||
expectedError: "cannot unmarshal",
|
||||
},
|
||||
}
|
||||
@ -59,7 +58,7 @@ func Test_Load(t *testing.T) {
|
||||
|
||||
func Test_LoadMetadata(t *testing.T) {
|
||||
t.Run("Metadata", func(t *testing.T) {
|
||||
dg, err := LoadMetadata(path.Join(testdataDir, "default.yaml"))
|
||||
dg, err := LoadMetadata(filepath.Join(testdataDir, "default.yaml"))
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, dg.Name, "default")
|
||||
@ -69,13 +68,8 @@ func Test_LoadMetadata(t *testing.T) {
|
||||
}
|
||||
|
||||
func Test_loadBaseConfig(t *testing.T) {
|
||||
t.Run("BaseConfigFile", func(t *testing.T) {
|
||||
// The base config file is set on the global config
|
||||
// This should be `testdata/home/.dagu/config.yaml`.
|
||||
cfg, err := config.Load()
|
||||
require.NoError(t, err)
|
||||
|
||||
dg, err := loadBaseConfig(cfg.BaseConfig, buildOpts{})
|
||||
t.Run("LoadBaseConfigFile", func(t *testing.T) {
|
||||
dg, err := loadBaseConfig(filepath.Join(testdataDir, "base.yaml"), buildOpts{})
|
||||
require.NotNil(t, dg)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
@ -83,7 +77,7 @@ func Test_loadBaseConfig(t *testing.T) {
|
||||
|
||||
func Test_LoadDefaultConfig(t *testing.T) {
|
||||
t.Run("DefaultConfigWithoutBaseConfig", func(t *testing.T) {
|
||||
file := path.Join(testdataDir, "default.yaml")
|
||||
file := filepath.Join(testdataDir, "default.yaml")
|
||||
dg, err := Load("", file, "")
|
||||
|
||||
require.NoError(t, err)
|
||||
@ -99,7 +93,7 @@ func Test_LoadDefaultConfig(t *testing.T) {
|
||||
require.Len(t, dg.Steps, 1)
|
||||
assert.Equal(t, "1", dg.Steps[0].Name, "1")
|
||||
assert.Equal(t, "true", dg.Steps[0].Command, "true")
|
||||
assert.Equal(t, path.Dir(file), dg.Steps[0].Dir)
|
||||
assert.Equal(t, filepath.Dir(file), dg.Steps[0].Dir)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -5,7 +5,7 @@ import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
@ -105,7 +105,7 @@ func TestStdout(t *testing.T) {
|
||||
|
||||
runTestNode(t, n)
|
||||
|
||||
f := path.Join(os.Getenv("HOME"), n.data.Step.Stdout)
|
||||
f := filepath.Join(os.Getenv("HOME"), n.data.Step.Stdout)
|
||||
dat, _ := os.ReadFile(f)
|
||||
require.Equal(t, "done\n", string(dat))
|
||||
}
|
||||
@ -127,11 +127,11 @@ echo Stderr message >&2
|
||||
|
||||
runTestNode(t, n)
|
||||
|
||||
f := path.Join(os.Getenv("HOME"), n.data.Step.Stderr)
|
||||
f := filepath.Join(os.Getenv("HOME"), n.data.Step.Stderr)
|
||||
dat, _ := os.ReadFile(f)
|
||||
require.Equal(t, "Stderr message\n", string(dat))
|
||||
|
||||
f = path.Join(os.Getenv("HOME"), n.data.Step.Stdout)
|
||||
f = filepath.Join(os.Getenv("HOME"), n.data.Step.Stdout)
|
||||
dat, _ = os.ReadFile(f)
|
||||
require.Equal(t, "Stdout message\n", string(dat))
|
||||
}
|
||||
|
||||
@ -3,7 +3,7 @@ package scheduler
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
@ -170,7 +170,7 @@ func TestSchedulerCancel(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSchedulerRetryFail(t *testing.T) {
|
||||
cmd := path.Join(util.MustGetwd(), "testdata/testfile.sh")
|
||||
cmd := filepath.Join(util.MustGetwd(), "testdata/testfile.sh")
|
||||
g, sc, err := testSchedule(t,
|
||||
dag.Step{
|
||||
Name: "1",
|
||||
@ -207,9 +207,9 @@ func TestSchedulerRetryFail(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSchedulerRetrySuccess(t *testing.T) {
|
||||
cmd := path.Join(util.MustGetwd(), "testdata/testfile.sh")
|
||||
cmd := filepath.Join(util.MustGetwd(), "testdata/testfile.sh")
|
||||
tmpDir, err := os.MkdirTemp("", "scheduler_test")
|
||||
tmpFile := path.Join(tmpDir, "flag")
|
||||
tmpFile := filepath.Join(tmpDir, "flag")
|
||||
|
||||
require.NoError(t, err)
|
||||
defer os.Remove(tmpDir)
|
||||
|
||||
@ -2,84 +2,29 @@ package engine_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/persistence"
|
||||
"github.com/dagu-dev/dagu/internal/persistence/client"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/config"
|
||||
"github.com/dagu-dev/dagu/internal/dag"
|
||||
"github.com/dagu-dev/dagu/internal/dag/scheduler"
|
||||
"github.com/dagu-dev/dagu/internal/engine"
|
||||
"github.com/dagu-dev/dagu/internal/persistence/model"
|
||||
"github.com/dagu-dev/dagu/internal/sock"
|
||||
"github.com/dagu-dev/dagu/internal/test"
|
||||
"github.com/dagu-dev/dagu/internal/util"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var testdataDir = path.Join(util.MustGetwd(), "./testdata")
|
||||
var lock sync.Mutex
|
||||
|
||||
func setupTest(t *testing.T) (
|
||||
string, engine.Engine, persistence.DataStores, *config.Config,
|
||||
) {
|
||||
t.Helper()
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
tmpDir := util.MustTempDir("dagu_test")
|
||||
_ = os.Setenv("HOME", tmpDir)
|
||||
cfg, _ := config.Load()
|
||||
|
||||
dataStore := client.NewDataStores(&client.NewDataStoresArgs{
|
||||
DataDir: path.Join(tmpDir, ".dagu", "data"),
|
||||
DAGs: testdataDir,
|
||||
})
|
||||
|
||||
exec := path.Join(util.MustGetwd(), "../../bin/dagu")
|
||||
|
||||
return tmpDir,
|
||||
engine.New(
|
||||
&engine.NewEngineArgs{DataStore: dataStore, Executable: exec},
|
||||
),
|
||||
dataStore, cfg
|
||||
}
|
||||
|
||||
func setupTestTmpDir(
|
||||
t *testing.T,
|
||||
) (string, engine.Engine, persistence.DataStores, *config.Config) {
|
||||
t.Helper()
|
||||
|
||||
tmpDir := util.MustTempDir("dagu_test")
|
||||
_ = os.Setenv("HOME", tmpDir)
|
||||
cfg, _ := config.Load()
|
||||
|
||||
dataStore := client.NewDataStores(&client.NewDataStoresArgs{
|
||||
DataDir: path.Join(tmpDir, ".dagu", "data"),
|
||||
DAGs: path.Join(tmpDir, ".dagu", "dags"),
|
||||
})
|
||||
|
||||
exec := path.Join(util.MustGetwd(), "../../bin/dagu")
|
||||
|
||||
return tmpDir,
|
||||
engine.New(
|
||||
&engine.NewEngineArgs{DataStore: dataStore, Executable: exec}),
|
||||
dataStore, cfg
|
||||
}
|
||||
var testdataDir = filepath.Join(util.MustGetwd(), "./testdata")
|
||||
|
||||
func TestEngine_GetStatus(t *testing.T) {
|
||||
t.Run("Valid", func(t *testing.T) {
|
||||
tmpDir, eng, _, _ := setupTest(t)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
file := testDAG("get_status.yaml")
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
file := testDAG("sleep1.yaml")
|
||||
|
||||
eng := setup.Engine()
|
||||
dagStatus, err := eng.GetStatus(file)
|
||||
require.NoError(t, err)
|
||||
|
||||
@ -112,10 +57,10 @@ func TestEngine_GetStatus(t *testing.T) {
|
||||
require.Equal(t, scheduler.StatusNone, curStatus.Status)
|
||||
})
|
||||
t.Run("InvalidDAGName", func(t *testing.T) {
|
||||
tmpDir, eng, _, _ := setupTest(t)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
eng := setup.Engine()
|
||||
|
||||
dagStatus, err := eng.GetStatus(testDAG("invalid_dag"))
|
||||
require.Error(t, err)
|
||||
@ -125,21 +70,20 @@ func TestEngine_GetStatus(t *testing.T) {
|
||||
require.Error(t, dagStatus.Error)
|
||||
})
|
||||
t.Run("UpdateStatus", func(t *testing.T) {
|
||||
tmpDir, eng, dataStore, _ := setupTest(t)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
var (
|
||||
file = testDAG("update_status.yaml")
|
||||
file = testDAG("success.yaml")
|
||||
requestID = "test-update-status"
|
||||
now = time.Now()
|
||||
eng = setup.Engine()
|
||||
)
|
||||
|
||||
dagStatus, err := eng.GetStatus(file)
|
||||
require.NoError(t, err)
|
||||
|
||||
historyStore := dataStore.HistoryStore()
|
||||
historyStore := setup.DataStore().HistoryStore()
|
||||
|
||||
err = historyStore.Open(dagStatus.DAG.Location, now, requestID)
|
||||
require.NoError(t, err)
|
||||
@ -170,13 +114,12 @@ func TestEngine_GetStatus(t *testing.T) {
|
||||
require.Equal(t, newStatus, statusByRequestID.Nodes[0].Status)
|
||||
})
|
||||
t.Run("InvalidUpdateStatusWithInvalidReqID", func(t *testing.T) {
|
||||
tmpDir, eng, _, _ := setupTest(t)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
var (
|
||||
file = testDAG("update_status_failed.yaml")
|
||||
eng = setup.Engine()
|
||||
file = testDAG("sleep1.yaml")
|
||||
wrongReqID = "invalid-request-id"
|
||||
)
|
||||
|
||||
@ -195,31 +138,28 @@ func TestEngine_GetStatus(t *testing.T) {
|
||||
|
||||
// nolint // paralleltest
|
||||
func TestEngine_RunDAG(t *testing.T) {
|
||||
t.Run("Start", func(t *testing.T) {
|
||||
tmpDir, eng, _, _ := setupTest(t)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
file := testDAG("start.yaml")
|
||||
t.Run("RunDAG", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
eng := setup.Engine()
|
||||
file := testDAG("success.yaml")
|
||||
dagStatus, err := eng.GetStatus(file)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = eng.Start(dagStatus.DAG, "")
|
||||
require.Error(t, err)
|
||||
require.NoError(t, err)
|
||||
|
||||
status, err := eng.GetLatestStatus(dagStatus.DAG)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, scheduler.StatusError.String(), status.Status.String())
|
||||
require.Equal(t, scheduler.StatusSuccess.String(), status.Status.String())
|
||||
})
|
||||
t.Run("Stop", func(t *testing.T) {
|
||||
tmpDir, eng, _, _ := setupTest(t)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
|
||||
file := testDAG("stop.yaml")
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
eng := setup.Engine()
|
||||
file := testDAG("sleep10.yaml")
|
||||
dagStatus, err := eng.GetStatus(file)
|
||||
require.NoError(t, err)
|
||||
|
||||
@ -238,13 +178,11 @@ func TestEngine_RunDAG(t *testing.T) {
|
||||
}, time.Millisecond*1500, time.Millisecond*100)
|
||||
})
|
||||
t.Run("Restart", func(t *testing.T) {
|
||||
tmpDir, eng, _, _ := setupTest(t)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
|
||||
file := testDAG("restart.yaml")
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
eng := setup.Engine()
|
||||
file := testDAG("success.yaml")
|
||||
dagStatus, err := eng.GetStatus(file)
|
||||
require.NoError(t, err)
|
||||
|
||||
@ -256,12 +194,11 @@ func TestEngine_RunDAG(t *testing.T) {
|
||||
require.Equal(t, scheduler.StatusSuccess, status.Status)
|
||||
})
|
||||
t.Run("Retry", func(t *testing.T) {
|
||||
tmpDir, eng, _, _ := setupTest(t)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
file := testDAG("retry.yaml")
|
||||
eng := setup.Engine()
|
||||
file := testDAG("success.yaml")
|
||||
|
||||
dagStatus, err := eng.GetStatus(file)
|
||||
require.NoError(t, err)
|
||||
@ -297,10 +234,10 @@ func TestEngine_RunDAG(t *testing.T) {
|
||||
func TestEngine_UpdateDAG(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("Update", func(t *testing.T) {
|
||||
tmpDir, eng, _, _ := setupTestTmpDir(t)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
eng := setup.Engine()
|
||||
|
||||
// valid DAG
|
||||
validDAG := `name: test DAG
|
||||
@ -326,10 +263,10 @@ steps:
|
||||
require.Equal(t, validDAG, spec)
|
||||
})
|
||||
t.Run("Remove", func(t *testing.T) {
|
||||
tmpDir, eng, _, _ := setupTestTmpDir(t)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
eng := setup.Engine()
|
||||
|
||||
spec := `name: test DAG
|
||||
steps:
|
||||
@ -353,30 +290,30 @@ steps:
|
||||
require.NoError(t, err)
|
||||
})
|
||||
t.Run("Create", func(t *testing.T) {
|
||||
tmpDir, eng, _, _ := setupTestTmpDir(t)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
eng := setup.Engine()
|
||||
|
||||
id, err := eng.CreateDAG("test-dag")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Check if the new DAG is actually created.
|
||||
dg, err := dag.Load("",
|
||||
path.Join(tmpDir, ".dagu", "dags", id+".yaml"), "")
|
||||
filepath.Join(setup.Config.DAGs, id+".yaml"), "")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "test-dag", dg.Name)
|
||||
})
|
||||
t.Run("Rename", func(t *testing.T) {
|
||||
tmpDir, eng, _, _ := setupTestTmpDir(t)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
eng := setup.Engine()
|
||||
|
||||
// Create a DAG to rename.
|
||||
id, err := eng.CreateDAG("old_name")
|
||||
require.NoError(t, err)
|
||||
_, err = eng.GetStatus(path.Join(tmpDir, ".dagu", "dags", id+".yaml"))
|
||||
_, err = eng.GetStatus(filepath.Join(setup.Config.DAGs, id+".yaml"))
|
||||
require.NoError(t, err)
|
||||
|
||||
// Rename the file.
|
||||
@ -384,44 +321,43 @@ steps:
|
||||
|
||||
// Check if the file is renamed.
|
||||
require.NoError(t, err)
|
||||
require.FileExists(t, path.Join(tmpDir, ".dagu", "dags", id+"_renamed.yaml"))
|
||||
require.FileExists(t, filepath.Join(setup.Config.DAGs, id+"_renamed.yaml"))
|
||||
})
|
||||
}
|
||||
|
||||
func TestEngin_ReadHistory(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("Read empty history", func(t *testing.T) {
|
||||
tmpDir, eng, _, _ := setupTest(t)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
func TestEngine_ReadHistory(t *testing.T) {
|
||||
t.Run("TestEngine_Empty", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
file := testDAG("read_status.yaml")
|
||||
eng := setup.Engine()
|
||||
file := testDAG("success.yaml")
|
||||
|
||||
_, err := eng.GetStatus(file)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
t.Run("Read all history", func(t *testing.T) {
|
||||
tmpDir, e, _, _ := setupTest(t)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}()
|
||||
t.Run("TestEngine_All", func(t *testing.T) {
|
||||
setup := test.SetupTest(t)
|
||||
defer setup.Cleanup()
|
||||
|
||||
allDagStatus, _, err := e.GetAllStatus()
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, len(allDagStatus), 0)
|
||||
eng := setup.Engine()
|
||||
|
||||
pattern := path.Join(testdataDir, "*.yaml")
|
||||
matches, err := filepath.Glob(pattern)
|
||||
// Create a DAG
|
||||
_, err := eng.CreateDAG("test-dag1")
|
||||
require.NoError(t, err)
|
||||
if len(matches) != len(allDagStatus) {
|
||||
t.Fatalf("unexpected number of dags: %d", len(allDagStatus))
|
||||
}
|
||||
|
||||
_, err = eng.CreateDAG("test-dag2")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get all statuses.
|
||||
allDagStatus, _, err := eng.GetAllStatus()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, len(allDagStatus))
|
||||
})
|
||||
}
|
||||
|
||||
func testDAG(name string) string {
|
||||
return path.Join(testdataDir, name)
|
||||
return filepath.Join(testdataDir, name)
|
||||
}
|
||||
|
||||
func testNewStatus(dg *dag.DAG, reqID string, status scheduler.Status,
|
||||
|
||||
3
internal/engine/testdata/read_status.yaml
vendored
3
internal/engine/testdata/read_status.yaml
vendored
@ -1,3 +0,0 @@
|
||||
steps:
|
||||
- name: "1"
|
||||
command: "true"
|
||||
3
internal/engine/testdata/restart.yaml
vendored
3
internal/engine/testdata/restart.yaml
vendored
@ -1,3 +0,0 @@
|
||||
steps:
|
||||
- name: "1"
|
||||
command: "true"
|
||||
4
internal/engine/testdata/retry.yaml
vendored
4
internal/engine/testdata/retry.yaml
vendored
@ -1,4 +0,0 @@
|
||||
params: "a b c"
|
||||
steps:
|
||||
- name: "1"
|
||||
command: "true"
|
||||
3
internal/engine/testdata/update_status.yaml
vendored
3
internal/engine/testdata/update_status.yaml
vendored
@ -1,3 +0,0 @@
|
||||
steps:
|
||||
- name: "1"
|
||||
command: "true"
|
||||
@ -1,3 +0,0 @@
|
||||
steps:
|
||||
- name: "1"
|
||||
command: "sleep 1"
|
||||
@ -56,7 +56,7 @@ func (o *DeleteDagParams) BindRequest(r *http.Request, route *middleware.Matched
|
||||
return nil
|
||||
}
|
||||
|
||||
// bindDagID binds and validates parameter DagID from path.
|
||||
// bindDagID binds and validates parameter DagID from filepath.
|
||||
func (o *DeleteDagParams) bindDagID(rawData []string, hasKey bool, formats strfmt.Registry) error {
|
||||
var raw string
|
||||
if len(rawData) > 0 {
|
||||
|
||||
@ -86,7 +86,7 @@ func (o *GetDagDetailsParams) BindRequest(r *http.Request, route *middleware.Mat
|
||||
return nil
|
||||
}
|
||||
|
||||
// bindDagID binds and validates parameter DagID from path.
|
||||
// bindDagID binds and validates parameter DagID from filepath.
|
||||
func (o *GetDagDetailsParams) bindDagID(rawData []string, hasKey bool, formats strfmt.Registry) error {
|
||||
var raw string
|
||||
if len(rawData) > 0 {
|
||||
|
||||
@ -84,7 +84,7 @@ func (o *PostDagActionParams) BindRequest(r *http.Request, route *middleware.Mat
|
||||
return nil
|
||||
}
|
||||
|
||||
// bindDagID binds and validates parameter DagID from path.
|
||||
// bindDagID binds and validates parameter DagID from filepath.
|
||||
func (o *PostDagActionParams) bindDagID(rawData []string, hasKey bool, formats strfmt.Registry) error {
|
||||
var raw string
|
||||
if len(rawData) > 0 {
|
||||
|
||||
@ -5,7 +5,7 @@ import (
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"text/template"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/constants"
|
||||
@ -19,7 +19,7 @@ var (
|
||||
func (srv *Server) useTemplate(
|
||||
layout string, name string,
|
||||
) func(http.ResponseWriter, any) {
|
||||
files := append(baseTemplates(), path.Join(templatePath, layout))
|
||||
files := append(baseTemplates(), filepath.Join(templatePath, layout))
|
||||
tmpl, err := template.New(name).Funcs(
|
||||
defaultFunctions(srv.funcsConfig)).ParseFS(srv.assets, files...,
|
||||
)
|
||||
@ -73,7 +73,7 @@ func baseTemplates() []string {
|
||||
var templateFiles = []string{"base.gohtml"}
|
||||
ret := make([]string, 0, len(templateFiles))
|
||||
for _, t := range templateFiles {
|
||||
ret = append(ret, path.Join(templatePath, t))
|
||||
ret = append(ret, filepath.Join(templatePath, t))
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
@ -28,8 +28,11 @@ type Cache[T any] struct {
|
||||
}
|
||||
|
||||
func New[T any](cap int, ttl time.Duration) *Cache[T] {
|
||||
c := &Cache[T]{capacity: cap, ttl: ttl}
|
||||
return c
|
||||
return &Cache[T]{
|
||||
capacity: cap,
|
||||
ttl: ttl,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache[T]) Stop() {
|
||||
@ -56,6 +59,7 @@ func (c *Cache[T]) evict() {
|
||||
entry := value.(Entry[T])
|
||||
if time.Now().After(entry.ExpiresAt) {
|
||||
c.entries.Delete(key)
|
||||
c.items.Add(-1)
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
@ -2,7 +2,7 @@ package grep
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
@ -11,7 +11,7 @@ import (
|
||||
func TestGrep(t *testing.T) {
|
||||
t.Parallel()
|
||||
wd, _ := os.Getwd()
|
||||
dir := path.Join(wd, "/testdata")
|
||||
dir := filepath.Join(wd, "/testdata")
|
||||
for _, tc := range []struct {
|
||||
Name string
|
||||
File string
|
||||
@ -22,7 +22,7 @@ func TestGrep(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
Name: "simple",
|
||||
File: path.Join(dir, "test.txt"),
|
||||
File: filepath.Join(dir, "test.txt"),
|
||||
Pattern: "b",
|
||||
Want: []*Match{
|
||||
{
|
||||
@ -33,7 +33,7 @@ func TestGrep(t *testing.T) {
|
||||
},
|
||||
{
|
||||
Name: "regexp",
|
||||
File: path.Join(dir, "test.txt"),
|
||||
File: filepath.Join(dir, "test.txt"),
|
||||
Pattern: "^b.",
|
||||
Opts: &Options{
|
||||
IsRegexp: true,
|
||||
@ -47,7 +47,7 @@ func TestGrep(t *testing.T) {
|
||||
},
|
||||
{
|
||||
Name: "before",
|
||||
File: path.Join(dir, "test.txt"),
|
||||
File: filepath.Join(dir, "test.txt"),
|
||||
Pattern: "b",
|
||||
Opts: &Options{
|
||||
Before: 1,
|
||||
@ -61,7 +61,7 @@ func TestGrep(t *testing.T) {
|
||||
},
|
||||
{
|
||||
Name: "before+after",
|
||||
File: path.Join(dir, "test.txt"),
|
||||
File: filepath.Join(dir, "test.txt"),
|
||||
Pattern: "cc",
|
||||
Opts: &Options{
|
||||
Before: 2,
|
||||
@ -76,7 +76,7 @@ func TestGrep(t *testing.T) {
|
||||
},
|
||||
{
|
||||
Name: "before+after,firstline",
|
||||
File: path.Join(dir, "test.txt"),
|
||||
File: filepath.Join(dir, "test.txt"),
|
||||
Pattern: "aa",
|
||||
Opts: &Options{
|
||||
Before: 1,
|
||||
@ -91,7 +91,7 @@ func TestGrep(t *testing.T) {
|
||||
},
|
||||
{
|
||||
Name: "before+after,lastline",
|
||||
File: path.Join(dir, "test.txt"),
|
||||
File: filepath.Join(dir, "test.txt"),
|
||||
Pattern: "ee",
|
||||
Opts: &Options{
|
||||
Before: 1,
|
||||
@ -106,25 +106,25 @@ func TestGrep(t *testing.T) {
|
||||
},
|
||||
{
|
||||
Name: "no match",
|
||||
File: path.Join(dir, "test.txt"),
|
||||
File: filepath.Join(dir, "test.txt"),
|
||||
Pattern: "no match text",
|
||||
IsErr: true,
|
||||
},
|
||||
{
|
||||
Name: "no file",
|
||||
File: path.Join(dir, "dummy.txt"),
|
||||
File: filepath.Join(dir, "dummy.txt"),
|
||||
Pattern: "aa",
|
||||
IsErr: true,
|
||||
},
|
||||
{
|
||||
Name: "no pattern",
|
||||
File: path.Join(dir, "test.txt"),
|
||||
File: filepath.Join(dir, "test.txt"),
|
||||
Pattern: "",
|
||||
IsErr: true,
|
||||
},
|
||||
{
|
||||
Name: "invalid regexp",
|
||||
File: path.Join(dir, "test.txt"),
|
||||
File: filepath.Join(dir, "test.txt"),
|
||||
Pattern: "(aa",
|
||||
Opts: &Options{
|
||||
IsRegexp: true,
|
||||
|
||||
@ -10,7 +10,6 @@ import (
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
@ -223,8 +222,8 @@ func (*Store) Compact(_, original string) error {
|
||||
}
|
||||
|
||||
newFile := fmt.Sprintf("%s_c.dat",
|
||||
strings.TrimSuffix(filepath.Base(original), path.Ext(original)))
|
||||
f := path.Join(filepath.Dir(original), newFile)
|
||||
strings.TrimSuffix(filepath.Base(original), filepath.Ext(original)))
|
||||
f := filepath.Join(filepath.Dir(original), newFile)
|
||||
w := &writer{target: f}
|
||||
if err := w.open(); err != nil {
|
||||
return err
|
||||
@ -270,12 +269,12 @@ func (s *Store) Rename(oldID, newID string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oldPrefix := path.Base(s.prefixWithDirectory(on))
|
||||
newPrefix := path.Base(s.prefixWithDirectory(nn))
|
||||
oldPrefix := filepath.Base(s.prefixWithDirectory(on))
|
||||
newPrefix := filepath.Base(s.prefixWithDirectory(nn))
|
||||
for _, m := range matches {
|
||||
base := path.Base(m)
|
||||
base := filepath.Base(m)
|
||||
f := strings.Replace(base, oldPrefix, newPrefix, 1)
|
||||
_ = os.Rename(m, path.Join(newDir, f))
|
||||
_ = os.Rename(m, filepath.Join(newDir, f))
|
||||
}
|
||||
if files, _ := os.ReadDir(oldDir); len(files) == 0 {
|
||||
_ = os.Remove(oldDir)
|
||||
@ -434,5 +433,5 @@ func readLineFrom(f *os.File, offset int64) ([]byte, error) {
|
||||
}
|
||||
|
||||
func prefix(dagFile string) string {
|
||||
return strings.TrimSuffix(filepath.Base(dagFile), path.Ext(dagFile))
|
||||
return strings.TrimSuffix(filepath.Base(dagFile), filepath.Ext(dagFile))
|
||||
}
|
||||
|
||||
@ -4,7 +4,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
@ -36,7 +35,7 @@ func TestNewDataFile(t *testing.T) {
|
||||
reqID := "request-id-1"
|
||||
f, err := db.newFile(d.Location, timestamp, reqID)
|
||||
require.NoError(t, err)
|
||||
p := util.ValidFilename(strings.TrimSuffix(path.Base(d.Location), path.Ext(d.Location)))
|
||||
p := util.ValidFilename(strings.TrimSuffix(filepath.Base(d.Location), filepath.Ext(d.Location)))
|
||||
require.Regexp(t, fmt.Sprintf("%s.*/%s.20220101.00:00:00.000.%s.dat", p, p, reqID[:8]), f)
|
||||
|
||||
_, err = db.newFile("", timestamp, reqID)
|
||||
|
||||
@ -3,7 +3,7 @@ package jsondb
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@ -24,7 +24,7 @@ type writer struct {
|
||||
|
||||
// Open opens the writer.
|
||||
func (w *writer) open() (err error) {
|
||||
_ = os.MkdirAll(path.Dir(w.target), 0755)
|
||||
_ = os.MkdirAll(filepath.Dir(w.target), 0755)
|
||||
w.file, err = util.OpenOrCreateFile(w.target)
|
||||
if err == nil {
|
||||
w.writer = bufio.NewWriter(w.file)
|
||||
|
||||
@ -79,9 +79,9 @@ func (er *entryReaderImpl) Read(now time.Time) ([]*entry, error) {
|
||||
if er.engine.IsSuspended(dg.Name) {
|
||||
continue
|
||||
}
|
||||
addEntriesFn(dg, dg.Schedule, Start)
|
||||
addEntriesFn(dg, dg.StopSchedule, Stop)
|
||||
addEntriesFn(dg, dg.RestartSchedule, Restart)
|
||||
addEntriesFn(dg, dg.Schedule, entryTypeStart)
|
||||
addEntriesFn(dg, dg.StopSchedule, entryTypeStop)
|
||||
addEntriesFn(dg, dg.RestartSchedule, entryTypeRestart)
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
|
||||
@ -2,7 +2,7 @@ package scheduler
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -25,7 +25,7 @@ func TestReadEntries(t *testing.T) {
|
||||
|
||||
now := time.Date(2020, 1, 1, 1, 0, 0, 0, time.UTC).Add(-time.Second)
|
||||
entryReader := newEntryReader(newEntryReaderArgs{
|
||||
DagsDir: path.Join(testdataDir, "invalid_directory"),
|
||||
DagsDir: filepath.Join(testdataDir, "invalid_directory"),
|
||||
JobCreator: &mockJobFactory{},
|
||||
Logger: logger.NewSlogLogger(),
|
||||
Engine: eng,
|
||||
@ -73,7 +73,7 @@ func TestReadEntries(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
var testdataDir = path.Join(util.MustGetwd(), "testdata")
|
||||
var testdataDir = filepath.Join(util.MustGetwd(), "testdata")
|
||||
|
||||
func setupTest(t *testing.T) (string, engine.Engine) {
|
||||
t.Helper()
|
||||
@ -84,7 +84,7 @@ func setupTest(t *testing.T) (string, engine.Engine) {
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg := &config.Config{
|
||||
DataDir: path.Join(tmpDir, ".dagu", "data"),
|
||||
DataDir: filepath.Join(tmpDir, ".dagu", "data"),
|
||||
DAGs: testdataDir,
|
||||
SuspendFlagsDir: tmpDir,
|
||||
}
|
||||
|
||||
@ -42,7 +42,7 @@ func LifetimeHooks(lc fx.Lifecycle, a *Scheduler) {
|
||||
lc.Append(
|
||||
fx.Hook{
|
||||
OnStart: func(ctx context.Context) (err error) {
|
||||
return a.Start()
|
||||
return a.Start(ctx)
|
||||
},
|
||||
OnStop: func(_ context.Context) error {
|
||||
a.Stop()
|
||||
|
||||
@ -1,10 +1,11 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@ -47,45 +48,45 @@ type job interface {
|
||||
type entryType int
|
||||
|
||||
const (
|
||||
Start entryType = iota
|
||||
Stop
|
||||
Restart
|
||||
entryTypeStart entryType = iota
|
||||
entryTypeStop
|
||||
entryTypeRestart
|
||||
)
|
||||
|
||||
func (e entryType) String() string {
|
||||
switch e {
|
||||
case entryTypeStart:
|
||||
return "start"
|
||||
case entryTypeStop:
|
||||
return "stop"
|
||||
case entryTypeRestart:
|
||||
return "restart"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
func (e *entry) Invoke() error {
|
||||
if e.Job == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
logMsg := fmt.Sprintf("%s job", e.EntryType)
|
||||
e.Logger.Info(logMsg,
|
||||
"job", e.Job.String(),
|
||||
"time", e.Next.Format(time.RFC3339),
|
||||
)
|
||||
|
||||
switch e.EntryType {
|
||||
case Start:
|
||||
e.Logger.Info(
|
||||
"start job",
|
||||
"job",
|
||||
e.Job.String(),
|
||||
"time",
|
||||
e.Next.Format("2006-01-02 15:04:05"),
|
||||
)
|
||||
case entryTypeStart:
|
||||
return e.Job.Start()
|
||||
case Stop:
|
||||
e.Logger.Info(
|
||||
"stop job",
|
||||
"job",
|
||||
e.Job.String(),
|
||||
"time",
|
||||
e.Next.Format("2006-01-02 15:04:05"),
|
||||
)
|
||||
case entryTypeStop:
|
||||
return e.Job.Stop()
|
||||
case Restart:
|
||||
e.Logger.Info(
|
||||
"restart job",
|
||||
"job",
|
||||
e.Job.String(),
|
||||
"time",
|
||||
e.Next.Format("2006-01-02 15:04:05"),
|
||||
)
|
||||
case entryTypeRestart:
|
||||
return e.Job.Restart()
|
||||
default:
|
||||
return fmt.Errorf("unknown entry type: %v", e.EntryType)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type newSchedulerArgs struct {
|
||||
@ -103,7 +104,7 @@ func newScheduler(args newSchedulerArgs) *Scheduler {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scheduler) Start() error {
|
||||
func (s *Scheduler) Start(ctx context.Context) error {
|
||||
if err := s.setupLogFile(); err != nil {
|
||||
return fmt.Errorf("setup log file: %w", err)
|
||||
}
|
||||
@ -124,6 +125,8 @@ func (s *Scheduler) Start() error {
|
||||
return
|
||||
case <-sig:
|
||||
s.Stop()
|
||||
case <-ctx.Done():
|
||||
s.Stop()
|
||||
}
|
||||
}()
|
||||
|
||||
@ -133,28 +136,33 @@ func (s *Scheduler) Start() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Scheduler) setupLogFile() (err error) {
|
||||
filename := path.Join(s.logDir, "scheduler.log")
|
||||
dir := path.Dir(filename)
|
||||
func (s *Scheduler) setupLogFile() error {
|
||||
filename := filepath.Join(s.logDir, "scheduler.log")
|
||||
dir := filepath.Dir(filename)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("create log directory: %w", err)
|
||||
}
|
||||
s.logger.Info("setup log", "filename", filename)
|
||||
return err
|
||||
s.logger.Info("Setup log", "filename", filename)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Scheduler) start() {
|
||||
t := now().Truncate(time.Second * 60)
|
||||
// TODO: refactor this to use a ticker
|
||||
t := now().Truncate(time.Minute)
|
||||
timer := time.NewTimer(0)
|
||||
|
||||
s.running.Store(true)
|
||||
for {
|
||||
select {
|
||||
case <-timer.C:
|
||||
s.run(t)
|
||||
t = s.nextTick(t)
|
||||
timer = time.NewTimer(t.Sub(now()))
|
||||
case <-s.stop:
|
||||
_ = timer.Stop()
|
||||
timer.Reset(t.Sub(now()))
|
||||
case <-s.stop:
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
@ -194,28 +202,29 @@ func (s *Scheduler) Stop() {
|
||||
return
|
||||
}
|
||||
if s.stop != nil {
|
||||
s.stop <- struct{}{}
|
||||
close(s.stop)
|
||||
}
|
||||
s.running.Store(false)
|
||||
s.logger.Info("Scheduler stopped")
|
||||
}
|
||||
|
||||
var (
|
||||
fixedTime time.Time
|
||||
lock sync.RWMutex
|
||||
timeLock sync.RWMutex
|
||||
)
|
||||
|
||||
// setFixedTime sets the fixed time.
|
||||
// This is used for testing.
|
||||
func setFixedTime(t time.Time) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
timeLock.Lock()
|
||||
defer timeLock.Unlock()
|
||||
fixedTime = t
|
||||
}
|
||||
|
||||
// now returns the current time.
|
||||
func now() time.Time {
|
||||
lock.RLock()
|
||||
defer lock.RUnlock()
|
||||
timeLock.RLock()
|
||||
defer timeLock.RUnlock()
|
||||
if fixedTime.IsZero() {
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -37,7 +38,7 @@ func TestScheduler(t *testing.T) {
|
||||
})
|
||||
|
||||
go func() {
|
||||
_ = schedulerInstance.Start()
|
||||
_ = schedulerInstance.Start(context.Background())
|
||||
}()
|
||||
|
||||
time.Sleep(time.Second + time.Millisecond*100)
|
||||
@ -53,7 +54,7 @@ func TestScheduler(t *testing.T) {
|
||||
entryReader := &mockEntryReader{
|
||||
Entries: []*entry{
|
||||
{
|
||||
EntryType: Restart,
|
||||
EntryType: entryTypeRestart,
|
||||
Job: &mockJob{},
|
||||
Next: now,
|
||||
Logger: logger.NewSlogLogger(),
|
||||
@ -68,7 +69,7 @@ func TestScheduler(t *testing.T) {
|
||||
})
|
||||
|
||||
go func() {
|
||||
_ = schedulerInstance.Start()
|
||||
_ = schedulerInstance.Start(context.Background())
|
||||
}()
|
||||
defer schedulerInstance.Stop()
|
||||
|
||||
|
||||
107
internal/test/setup.go
Normal file
107
internal/test/setup.go
Normal file
@ -0,0 +1,107 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/dagu-dev/dagu/internal/config"
|
||||
"github.com/dagu-dev/dagu/internal/engine"
|
||||
"github.com/dagu-dev/dagu/internal/persistence"
|
||||
"github.com/dagu-dev/dagu/internal/persistence/client"
|
||||
"github.com/dagu-dev/dagu/internal/util"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type Setup struct {
|
||||
Config *config.Config
|
||||
|
||||
homeDir string
|
||||
}
|
||||
|
||||
func (t Setup) Cleanup() {
|
||||
_ = os.RemoveAll(t.homeDir)
|
||||
}
|
||||
|
||||
func (t Setup) DataStore() persistence.DataStores {
|
||||
return client.NewDataStores(&client.NewDataStoresArgs{
|
||||
DAGs: t.Config.DAGs,
|
||||
DataDir: t.Config.DataDir,
|
||||
SuspendFlagsDir: t.Config.SuspendFlagsDir,
|
||||
LatestStatusToday: t.Config.LatestStatusToday,
|
||||
})
|
||||
}
|
||||
|
||||
func (t Setup) Engine() engine.Engine {
|
||||
return engine.New(&engine.NewEngineArgs{
|
||||
DataStore: t.DataStore(),
|
||||
Executable: t.Config.Executable,
|
||||
WorkDir: t.Config.WorkDir,
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
var (
|
||||
lock sync.Mutex
|
||||
)
|
||||
|
||||
func SetupTest(t *testing.T) Setup {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
tmpDir := util.MustTempDir("dagu_test")
|
||||
err := os.Setenv("HOME", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
viper.AddConfigPath(config.ConfigDir)
|
||||
viper.SetConfigType("yaml")
|
||||
viper.SetConfigName("admin")
|
||||
|
||||
config.ConfigDir = filepath.Join(tmpDir, "config")
|
||||
config.DataDir = filepath.Join(tmpDir, "data")
|
||||
config.LogsDir = filepath.Join(tmpDir, "log")
|
||||
|
||||
cfg, err := config.Load()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Set the executable path to the test binary.
|
||||
cfg.Executable = filepath.Join(util.MustGetwd(), "../../bin/dagu")
|
||||
|
||||
// Set environment variables.
|
||||
// This is required for some tests that run the executable
|
||||
_ = os.Setenv("DAGU_DAGS_DIR", cfg.DAGs)
|
||||
_ = os.Setenv("DAGU_WORK_DIR", cfg.WorkDir)
|
||||
_ = os.Setenv("DAGU_BASE_CONFIG", cfg.BaseConfig)
|
||||
_ = os.Setenv("DAGU_LOG_DIR", cfg.LogDir)
|
||||
_ = os.Setenv("DAGU_DATA_DIR", cfg.DataDir)
|
||||
_ = os.Setenv("DAGU_SUSPEND_FLAGS_DIR", cfg.SuspendFlagsDir)
|
||||
_ = os.Setenv("DAGU_ADMIN_LOG_DIR", cfg.AdminLogsDir)
|
||||
|
||||
return Setup{
|
||||
Config: cfg,
|
||||
homeDir: tmpDir,
|
||||
}
|
||||
}
|
||||
|
||||
func SetupForDir(t *testing.T, dir string) Setup {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
tmpDir := util.MustTempDir("dagu_test")
|
||||
err := os.Setenv("HOME", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
viper.AddConfigPath(dir)
|
||||
viper.SetConfigType("yaml")
|
||||
viper.SetConfigName("admin")
|
||||
|
||||
cfg, err := config.Load()
|
||||
require.NoError(t, err)
|
||||
|
||||
return Setup{
|
||||
Config: cfg,
|
||||
homeDir: tmpDir,
|
||||
}
|
||||
}
|
||||
@ -6,7 +6,7 @@ import (
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -136,7 +136,7 @@ func Test_OpenOrCreateFile(t *testing.T) {
|
||||
tmp, err := os.MkdirTemp("", "open_or_create")
|
||||
require.NoError(t, err)
|
||||
|
||||
name := path.Join(tmp, "/file.txt")
|
||||
name := filepath.Join(tmp, "/file.txt")
|
||||
f, err := util.OpenOrCreateFile(name)
|
||||
require.NoError(t, err)
|
||||
|
||||
@ -155,7 +155,7 @@ func Test_OpenOrCreateFile(t *testing.T) {
|
||||
_ = os.RemoveAll(dir)
|
||||
}()
|
||||
|
||||
filename := path.Join(dir, "test.txt")
|
||||
filename := filepath.Join(dir, "test.txt")
|
||||
createdFile, err := util.OpenOrCreateFile(filename)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
|
||||
1
tools.go
1
tools.go
@ -9,5 +9,6 @@ import (
|
||||
_ "github.com/go-swagger/go-swagger/cmd/swagger"
|
||||
_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
|
||||
_ "github.com/segmentio/golines"
|
||||
_ "github.com/yohamta/gomerger"
|
||||
_ "gotest.tools/gotestsum"
|
||||
)
|
||||
|
||||
@ -1,4 +1,12 @@
|
||||
import { DAG, DAGStatus, Node, NodeStatus, Schedule, SchedulerStatus, StatusFile } from './index';
|
||||
import {
|
||||
DAG,
|
||||
DAGStatus,
|
||||
Node,
|
||||
NodeStatus,
|
||||
Schedule,
|
||||
SchedulerStatus,
|
||||
StatusFile,
|
||||
} from './index';
|
||||
|
||||
export type GetDAGResponse = {
|
||||
Title: string;
|
||||
@ -47,8 +55,8 @@ export type GridData = {
|
||||
};
|
||||
|
||||
export type ListWorkflowsResponse = {
|
||||
DAGs: WorkflowListItem[];
|
||||
Errors: string[];
|
||||
DAGs?: WorkflowListItem[];
|
||||
Errors?: string[];
|
||||
HasError: boolean;
|
||||
};
|
||||
|
||||
@ -81,4 +89,4 @@ export type WorkflowStatus = {
|
||||
FinishedAt: string;
|
||||
Log: string;
|
||||
Params: string;
|
||||
};
|
||||
};
|
||||
|
||||
@ -32,7 +32,7 @@ function DAGs() {
|
||||
|
||||
const merged = React.useMemo(() => {
|
||||
const ret: DAGItem[] = [];
|
||||
if (data) {
|
||||
if (data && data.DAGs) {
|
||||
for (const val of data.DAGs) {
|
||||
if (!val.ErrorT) {
|
||||
ret.push({
|
||||
@ -72,8 +72,8 @@ function DAGs() {
|
||||
{data && (
|
||||
<React.Fragment>
|
||||
<DAGErrors
|
||||
DAGs={data.DAGs}
|
||||
errors={data.Errors}
|
||||
DAGs={data.DAGs || []}
|
||||
errors={data.Errors || []}
|
||||
hasError={data.HasError}
|
||||
></DAGErrors>
|
||||
<DAGTable
|
||||
|
||||
@ -31,7 +31,7 @@ function Dashboard() {
|
||||
return;
|
||||
}
|
||||
const m = { ...defaultMetrics };
|
||||
data.DAGs.forEach((wf) => {
|
||||
data.DAGs?.forEach((wf) => {
|
||||
if (wf.Status && wf.Status.Status) {
|
||||
const status = wf.Status.Status;
|
||||
m[status] += 1;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user