Compliance with XDG (#619)

This commit is contained in:
Yota Hamada 2024-07-19 00:40:36 +09:00 committed by GitHub
parent 3a6bf29bfe
commit 38695a2f86
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
79 changed files with 863 additions and 817 deletions

6
.gitignore vendored
View File

@ -19,7 +19,5 @@ tmp/*
# Goland
.idea
# local development
!local/cert
local/cert/*
!local/cert/openssl.conf
# Directory for local development
local/

View File

@ -45,10 +45,10 @@ WORKDIR /home/${USER}
COPY --from=go-builder /app/bin/dagu /usr/local/bin/
RUN mkdir -p .dagu/dags
RUN mkdir -p .config/dagu/dags
# Add the hello_world.yaml file
COPY <<EOF .dagu/dags/hello_world.yaml
COPY <<EOF .config/dagu/dags/hello_world.yaml
schedule: "* * * * *"
steps:
- name: hello world

191
Makefile
View File

@ -1,99 +1,130 @@
.PHONY: build server scheduler test proto certs swagger https
.PHONY: run run-server run-server-https run-scheduler test lint build certs swagger
########## Arguments ##########
##############################################################################
# Arguments
##############################################################################
VERSION=
########## Variables ##########
##############################################################################
# Variables
##############################################################################
# This Makefile's directory
SCRIPT_DIR=$(abspath $(dir $(lastword $(MAKEFILE_LIST))))
# Directories for miscellaneous files for the local environment
LOCAL_DIR=$(SCRIPT_DIR)/local
LOCAL_BIN_DIR=$(LOCAL_DIR)/bin
SRC_DIR=$(SCRIPT_DIR)
DST_DIR=$(SRC_DIR)/internal
# Configuration directory
CONFIG_DIR=$(SCRIPT_DIR)/config
# Local build settings
BIN_DIR=$(SCRIPT_DIR)/bin
BUILD_VERSION=$(shell date +'%y%m%d%H%M%S')
LDFLAGS=-X 'main.version=$(BUILD_VERSION)'
# Application name
APP_NAME=dagu
# Docker image build configuration
DOCKER_CMD := docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7,linux/arm64/v8 --builder container --build-arg VERSION=$(VERSION) --push --no-cache
# Arguments for the tests
GOTESTSUM_ARGS=--format=standard-quiet
GO_TEST_FLAGS=-v --race
# Frontend directories
FE_DIR=./internal/frontend
FE_GEN_DIR=${FE_DIR}/gen
FE_ASSETS_DIR=${FE_DIR}/assets
FE_BUILD_DIR=./ui/dist
FE_BUNDLE_JS=${FE_ASSETS_DIR}/bundle.js
# Colors for the output
COLOR_GREEN=\033[0;32m
COLOR_RESET=\033[0m
COLOR_RED=\033[0;31m
# Go packages for the tools
PKG_swagger=github.com/go-swagger/go-swagger/cmd/swagger
PKG_golangci_lint=github.com/golangci/golangci-lint/cmd/golangci-lint
PKG_gotestsum=gotest.tools/gotestsum
PKG_gomerger=github.com/yohamta/gomerger
# Certificates for the development environment
CERTS_DIR=${LOCAL_DIR}/certs
DEV_CERT_SUBJ_CA="/C=TR/ST=ASIA/L=TOKYO/O=DEV/OU=DAGU/CN=*.dagu.dev/emailAddress=ca@dev.com"
DEV_CERT_SUBJ_SERVER="/C=TR/ST=ASIA/L=TOKYO/O=DEV/OU=SERVER/CN=*.server.dev/emailAddress=server@dev.com"
DEV_CERT_SUBJ_CLIENT="/C=TR/ST=ASIA/L=TOKYO/O=DEV/OU=CLIENT/CN=*.client.dev/emailAddress=client@dev.com"
DEV_CERT_SUBJ_ALT="subjectAltName=DNS:localhost"
PKG_SWAGGER=github.com/go-swagger/go-swagger/cmd/swagger
PKG_GOLANGCI_LINT=github.com/golangci/golangci-lint/cmd/golangci-lint
PKG_gotestsum=gotest.tools/gotestsum
CA_CERT_FILE=${CERTS_DIR}/ca-cert.pem
CA_KEY_FILE=${CERTS_DIR}/ca-key.pem
SERVER_CERT_REQ=${CERTS_DIR}/server-req.pem
SERVER_CERT_FILE=${CERTS_DIR}/server-cert.pem
SERVER_KEY_FILE=${CERTS_DIR}/server-key.pem
CLIENT_CERT_REQ=${CERTS_DIR}/client-req.pem
CLIENT_CERT_FILE=${CERTS_DIR}/client-cert.pem
CLIENT_KEY_FILE=${CERTS_DIR}/client-key.pem
OPENSSL_CONF=${CONFIG_DIR}/openssl.local.conf
COLOR_GREEN=\033[0;32m
COLOR_RESET=\033[0m
FE_DIR=./internal/frontend
FE_GEN_DIR=${FE_DIR}/gen
FE_ASSETS_DIR=${FE_DIR}/assets
CERT_DIR=${LOCAL_DIR}/cert
CA_CERT_FILE=${CERT_DIR}/ca-cert.pem
CA_KEY_FILE=${CERT_DIR}/ca-key.pem
SERVER_CERT_REQ=${CERT_DIR}/server-req.pem
SERVER_CERT_FILE=${CERT_DIR}/server-cert.pem
SERVER_KEY_FILE=${CERT_DIR}/server-key.pem
CLIENT_CERT_REQ=${CERT_DIR}/client-req.pem
CLIENT_CERT_FILE=${CERT_DIR}/client-cert.pem
CLIENT_KEY_FILE=${CERT_DIR}/client-key.pem
OPENSSL_CONF=${CERT_DIR}/openssl.conf
FE_BUILD_DIR=./ui/dist
FE_BUNDLE_JS=${FE_ASSETS_DIR}/bundle.js
APP_NAME=dagu
BIN_DIR=${SCRIPT_DIR}/bin
# gotestsum args
GOTESTSUM_ARGS=--format=standard-quiet
GO_TEST_FLAGS=-v --race
########## Main Targets ##########
##############################################################################
# Targets
##############################################################################
# run starts the frontend server and the scheduler.
run: ${FE_BUNDLE_JS}
go run . start-all
@echo "${COLOR_GREEN}Starting the frontend server and the scheduler...${COLOR_RESET}"
@go run . start-all
# server build the binary and start the server.
run-server: golangci-lint build-bin
@echo "${COLOR_GREEN}Starting the server...${COLOR_RESET}"
${LOCAL_BIN_DIR}/${APP_NAME} server
# scheduler build the binary and start the scheduler.
run-scheduler: golangci-lint build-bin
@echo "${COLOR_GREEN}Starting the scheduler...${COLOR_RESET}"
${LOCAL_BIN_DIR}/${APP_NAME} scheduler
# check if the frontend assets are built.
${FE_BUNDLE_JS}:
echo "Please run 'make build-ui' to build the frontend assets."
@echo "${COLOR_RED}Error: frontend assets are not built.${COLOR_RESET}"
@echo "${COLOR_RED}Please run 'make build-ui' before starting the server.${COLOR_RESET}"
# https starts the server with the HTTPS protocol.
https: ${SERVER_CERT_FILE} ${SERVER_KEY_FILE}
run-server-https: ${SERVER_CERT_FILE} ${SERVER_KEY_FILE}
@echo "${COLOR_GREEN}Starting the server with HTTPS...${COLOR_RESET}"
@DAGU_CERT_FILE=${SERVER_CERT_FILE} \
DAGU_KEY_FILE=${SERVER_KEY_FILE} \
go run . start-all
# watch starts development UI server.
# The backend server should be running.
watch:
@echo "${COLOR_GREEN}Installing nodemon...${COLOR_RESET}"
@npm install -g nodemon
@nodemon --watch . --ext go,gohtml --verbose --signal SIGINT --exec 'make server'
# test runs all tests.
test:
@go install ${PKG_gotestsum}
@gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} ./...
@echo "${COLOR_GREEN}Running tests...${COLOR_RESET}"
@GOBIN=${LOCAL_BIN_DIR} go install ${PKG_gotestsum}
@${LOCAL_BIN_DIR}/gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} ./...
# test-coverage runs all tests with coverage.
test-coverage:
@go install ${PKG_gotestsum}
@gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} -coverprofile="coverage.txt" -covermode=atomic ./...
@echo "${COLOR_GREEN}Running tests with coverage...${COLOR_RESET}"
@GOBIN=${LOCAL_BIN_DIR} go install ${PKG_gotestsum}
@${LOCAL_BIN_DIR}/gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} -coverprofile="coverage.txt" -covermode=atomic ./...
# test-clean cleans the test cache and run all tests.
test-clean: build-bin
@go install ${PKG_gotestsum}
@echo "${COLOR_GREEN}Running tests...${COLOR_RESET}"
@GOBIN=${LOCAL_BIN_DIR} go install ${PKG_gotestsum}
@go clean -testcache
@gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} ./...
@${LOCAL_BIN_DIR}/gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} ./...
# lint runs the linter.
lint: golangci-lint
@ -102,7 +133,7 @@ lint: golangci-lint
swagger: clean-swagger gen-swagger
# certs generates the certificates to use in the development environment.
certs: ${SERVER_CERT_FILE} ${CLIENT_CERT_FILE} gencert-check
certs: ${CERTS_DIR} ${SERVER_CERT_FILE} ${CLIENT_CERT_FILE} certs-check
# build build the binary.
build: build-ui build-bin
@ -112,30 +143,38 @@ build: build-ui build-bin
# ```sh
# make build-image VERSION={version}
# ```
# {version} should be the version number such as v1.13.0.
# {version} should be the version number such as "1.13.0".
build-image: build-image-version build-image-latest
build-image-version:
ifeq ($(VERSION),)
$(error "VERSION is null")
$(error "VERSION is not set")
endif
echo "${COLOR_GREEN}Building the docker image with the version $(VERSION)...${COLOR_RESET}"
$(DOCKER_CMD) -t ghcr.io/dagu-dev/${APP_NAME}:$(VERSION) .
# build-image-latest build the docker image with the latest tag and push to
# the registry.
build-image-latest:
@echo "${COLOR_GREEN}Building the docker image...${COLOR_RESET}"
$(DOCKER_CMD) -t ghcr.io/dagu-dev/${APP_NAME}:latest .
# server build the binary and start the server.
server: golangci-lint build-bin
${BIN_DIR}/${APP_NAME} server
gomerger: ${LOCAL_DIR}/merged
@echo "${COLOR_GREEN}Merging Go files...${COLOR_RESET}"
@rm -f ${LOCAL_DIR}/merged/merged_project.go
@GOBIN=${LOCAL_BIN_DIR} go install ${PKG_gomerger}
@${LOCAL_BIN_DIR}/gomerger .
@mv merged_project.go ${LOCAL_DIR}/merged/
# scheduler build the binary and start the scheduler.
scheduler: golangci-lint build-bin
${BIN_DIR}/${APP_NAME} scheduler
${LOCAL_DIR}/merged:
@mkdir -p ${LOCAL_DIR}/merged
########## Tools ##########
##############################################################################
# Internal targets
##############################################################################
build-bin:
@echo "${COLOR_GREEN}Building the binary...${COLOR_RESET}"
@mkdir -p ${BIN_DIR}
@go build -ldflags="$(LDFLAGS)" -o ${BIN_DIR}/${APP_NAME} .
@ -148,20 +187,25 @@ build-ui:
@cp ${FE_BUILD_DIR}/* ${FE_ASSETS_DIR}
golangci-lint:
@go install $(PKG_GOLANGCI_LINT)
@golangci-lint run ./...
@echo "${COLOR_GREEN}Running linter...${COLOR_RESET}"
@GOBIN=${LOCAL_BIN_DIR} go install $(PKG_golangci_lint)
@${LOCAL_BIN_DIR}/golangci-lint run ./...
clean-swagger:
@echo "${COLOR_GREEN}Cleaning the swagger files...${COLOR_RESET}"
@rm -rf ${FE_GEN_DIR}/restapi/models
@rm -rf ${FE_GEN_DIR}/restapi/operations
gen-swagger:
@go install $(PKG_SWAGGER)
@swagger validate ./swagger.yaml
@swagger generate server -t ${FE_GEN_DIR} --server-package=restapi --exclude-main -f ./swagger.yaml
@echo "${COLOR_GREEN}Generating the swagger server code...${COLOR_RESET}"
@GOBIN=${LOCAL_BIN_DIR} go install $(PKG_swagger)
@${LOCAL_BIN_DIR}/swagger validate ./swagger.yaml
@${LOCAL_BIN_DIR}/swagger generate server -t ${FE_GEN_DIR} --server-package=restapi --exclude-main -f ./swagger.yaml
@go mod tidy
########## Certificates ##########
##############################################################################
# Certificates
##############################################################################
${CA_CERT_FILE}:
@echo "${COLOR_GREEN}Generating CA certificates...${COLOR_RESET}"
@ -194,7 +238,10 @@ ${CLIENT_CERT_FILE}: ${CA_CERT_FILE} ${CLIENT_KEY_FILE}
-CAkey ${CA_KEY_FILE} -CAcreateserial -out ${CLIENT_CERT_FILE} \
-extfile ${OPENSSL_CONF}
gencert-check:
${CERTS_DIR}:
@echo "${COLOR_GREEN}Creating the certificates directory...${COLOR_RESET}"
@mkdir -p ${CERTS_DIR}
certs-check:
@echo "${COLOR_GREEN}Checking CA certificate...${COLOR_RESET}"
@openssl x509 -in ${SERVER_CERT_FILE} -noout -text

View File

@ -67,6 +67,7 @@ Dagu is a powerful Cron alternative that comes with a Web UI. It allows you to d
- [**Documentation**](#documentation)
- [**Running as a daemon**](#running-as-a-daemon)
- [**Example DAG**](#example-dag)
- [**Docker-compose setting**](#docker-compose-setting)
- [**Motivation**](#motivation)
- [**Why Not Use an Existing DAG Scheduler Like Airflow?**](#why-not-use-an-existing-dag-scheduler-like-airflow)
- [**How It Works**](#how-it-works)
@ -176,12 +177,13 @@ brew upgrade dagu-dev/brew/dagu
docker run \
--rm \
-p 8080:8080 \
-v $HOME/.dagu/dags:/home/dagu/.dagu/dags \
-v $HOME/.dagu/data:/home/dagu/.dagu/data \
-v $HOME/.dagu/logs:/home/dagu/.dagu/logs \
-v $HOME/.config/dagu/dags:/home/dagu/.config/dagu/dags \
-v $HOME/.local/share/dagu:/home/dagu/.local/share/dagu \
ghcr.io/dagu-dev/dagu:latest dagu start-all
```
See [Environment variables](https://dagu.readthedocs.io/en/latest/config.html#environment-variables) to configure those default directories.
## **Quick Start Guide**
### 1. Launch the Web UI
@ -192,7 +194,7 @@ Start the server and scheduler with the command `dagu start-all` and browse to `
Navigate to the DAG List page by clicking the menu in the left panel of the Web UI. Then create a DAG by clicking the `NEW` button at the top of the page. Enter `example` in the dialog.
_Note: DAG (YAML) files will be placed in `~/.dagu/dags` by default. See [Configuration Options](https://dagu.readthedocs.io/en/latest/config.html) for more details._
_Note: DAG (YAML) files will be placed in `~/.config/dagu/dags` by default. See [Configuration Options](https://dagu.readthedocs.io/en/latest/config.html) for more details._
### 3. Edit the DAG
@ -387,6 +389,10 @@ steps:
- send_report
```
## **Docker-compose setting**
To run Dagu using Docker-compose, please take a look at the example: [docker-compose file](examples/docker-compose.yaml)
## **Motivation**
Legacy systems often have complex and implicit dependencies between jobs. When there are hundreds of cron jobs on a server, it can be difficult to keep track of these dependencies and to determine which job to rerun if one fails. It can also be a hassle to SSH into a server to view logs and manually rerun shell scripts one by one. Dagu aims to solve these problems by allowing you to explicitly visualize and manage pipeline dependencies as a DAG, and by providing a web UI for checking dependencies, execution status, and logs and for rerunning or stopping jobs with a simple mouse click.

View File

@ -1,8 +1,7 @@
package cmd
import (
"os"
"path"
"path/filepath"
"testing"
"time"
@ -17,43 +16,6 @@ import (
"github.com/stretchr/testify/require"
)
type testSetup struct {
homeDir string
engine engine.Engine
dataStore persistence.DataStores
cfg *config.Config
}
func (t testSetup) cleanup() {
_ = os.RemoveAll(t.homeDir)
}
// setupTest is a helper function to setup the test environment.
// This function does the following:
// 1. It creates a temporary directory and returns the path to it.
// 2. Sets the home directory to the temporary directory.
// 3. Creates a new data store factory and engine.
func setupTest(t *testing.T) testSetup {
t.Helper()
tmpDir := util.MustTempDir("dagu_test")
err := os.Setenv("HOME", tmpDir)
require.NoError(t, err)
cfg, err := config.Load()
require.NoError(t, err)
cfg.DataDir = path.Join(tmpDir, ".dagu", "data")
dataStore := newDataStores(cfg)
return testSetup{
homeDir: tmpDir,
dataStore: dataStore,
engine: newEngine(cfg),
cfg: cfg,
}
}
// cmdTest is a helper struct to test commands.
// It contains the arguments to the command and the expected output.
type cmdTest struct {
@ -128,8 +90,8 @@ func withSpool(t *testing.T, testFunction func()) string {
*/
func testDAGFile(name string) string {
return path.Join(
path.Join(util.MustGetwd(), "testdata"),
return filepath.Join(
filepath.Join(util.MustGetwd(), "testdata"),
name,
)
}

View File

@ -2,16 +2,18 @@ package cmd
import (
"testing"
"github.com/dagu-dev/dagu/internal/test"
)
func TestDryCommand(t *testing.T) {
t.Run("Dry-run command should run", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("DryRun", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
tests := []cmdTest{
{
args: []string{"dry", testDAGFile("dry.yaml")},
args: []string{"dry", testDAGFile("success.yaml")},
expectedOut: []string{"Starting DRY-RUN"},
},
}

View File

@ -6,6 +6,7 @@ import (
"github.com/dagu-dev/dagu/internal/dag"
"github.com/dagu-dev/dagu/internal/dag/scheduler"
"github.com/dagu-dev/dagu/internal/test"
"github.com/stretchr/testify/require"
)
@ -14,9 +15,9 @@ const (
)
func TestRestartCommand(t *testing.T) {
t.Run("Restart a DAG", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("RestartDAG", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
dagFile := testDAGFile("restart.yaml")
@ -30,7 +31,7 @@ func TestRestartCommand(t *testing.T) {
}()
time.Sleep(waitForStatusUpdate)
eng := setup.engine
eng := setup.Engine()
// Wait for the DAG running.
testStatusEventual(t, eng, dagFile, scheduler.StatusRunning)
@ -56,10 +57,10 @@ func TestRestartCommand(t *testing.T) {
testStatusEventual(t, eng, dagFile, scheduler.StatusNone)
// Check parameter was the same as the first execution
dg, err := dag.Load(setup.cfg.BaseConfig, dagFile, "")
dg, err := dag.Load(setup.Config.BaseConfig, dagFile, "")
require.NoError(t, err)
recentHistory := newEngine(setup.cfg).GetRecentHistory(dg, 2)
recentHistory := newEngine(setup.Config).GetRecentHistory(dg, 2)
require.Len(t, recentHistory, 2)
require.Equal(t, recentHistory[0].Status.Params, recentHistory[1].Status.Params)

View File

@ -5,13 +5,14 @@ import (
"testing"
"github.com/dagu-dev/dagu/internal/dag/scheduler"
"github.com/dagu-dev/dagu/internal/test"
"github.com/stretchr/testify/require"
)
func TestRetryCommand(t *testing.T) {
t.Run("Retry a DAG", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("RetryDAG", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
dagFile := testDAGFile("retry.yaml")
@ -19,7 +20,7 @@ func TestRetryCommand(t *testing.T) {
testRunCommand(t, startCmd(), cmdTest{args: []string{"start", `--params="foo"`, dagFile}})
// Find the request ID.
eng := setup.engine
eng := setup.Engine()
status, err := eng.GetStatus(dagFile)
require.NoError(t, err)
require.Equal(t, status.Status.Status, scheduler.StatusSuccess)

View File

@ -4,9 +4,7 @@ Copyright © 2023 NAME HERE <EMAIL ADDRESS>
package cmd
import (
"os"
"path"
"github.com/dagu-dev/dagu/internal/config"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
@ -23,8 +21,6 @@ var (
}
)
const configPath = ".dagu"
// Execute adds all child commands to the root command and sets flags
// appropriately. This is called by main.main(). It only needs to happen
// once to the rootCmd.
@ -32,14 +28,6 @@ func Execute() error {
return rootCmd.Execute()
}
func setDefaultConfigPath() {
homeDir, err := os.UserHomeDir()
if err != nil {
panic("could not determine home directory")
}
viper.AddConfigPath(path.Join(homeDir, configPath))
}
func registerCommands() {
rootCmd.AddCommand(startCmd())
rootCmd.AddCommand(stopCmd())
@ -57,7 +45,7 @@ func init() {
rootCmd.PersistentFlags().
StringVar(
&cfgFile, "config", "",
"config file (default is $HOME/.dagu/admin.yaml)",
"config file (default is $HOME/.config/dagu/admin.yaml)",
)
cobra.OnInitialize(initialize)
@ -71,7 +59,7 @@ func initialize() {
return
}
setDefaultConfigPath()
viper.AddConfigPath(config.ConfigDir)
viper.SetConfigType("yaml")
viper.SetConfigName("admin")
}

View File

@ -39,7 +39,7 @@ func schedulerCmd() *cobra.Command {
}
cmd.Flags().StringP(
"dags", "d", "", "location of DAG files (default is $HOME/.dagu/dags)",
"dags", "d", "", "location of DAG files (default is $HOME/.config/dagu/dags)",
)
_ = viper.BindPFlag("dags", cmd.Flags().Lookup("dags"))

View File

@ -3,12 +3,14 @@ package cmd
import (
"testing"
"time"
"github.com/dagu-dev/dagu/internal/test"
)
func TestSchedulerCommand(t *testing.T) {
t.Run("Start the scheduler", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("StartScheduler", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
go func() {
testRunCommand(t, schedulerCmd(), cmdTest{

View File

@ -47,7 +47,7 @@ func serverCmd() *cobra.Command {
func bindServerCommandFlags(cmd *cobra.Command) {
cmd.Flags().StringP(
"dags", "d", "", "location of DAG files (default is $HOME/.dagu/dags)",
"dags", "d", "", "location of DAG files (default is $HOME/.config/dagu/dags)",
)
cmd.Flags().StringP("host", "s", "", "server host (default is localhost)")
cmd.Flags().StringP("port", "p", "", "server port (default is 8080)")

View File

@ -6,13 +6,14 @@ import (
"testing"
"time"
"github.com/dagu-dev/dagu/internal/test"
"github.com/stretchr/testify/require"
)
func TestServerCommand(t *testing.T) {
t.Run("Start the server", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("StartServer", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
go func() {
testRunCommand(t, serverCmd(), cmdTest{

View File

@ -69,7 +69,7 @@ func startAllCmd() *cobra.Command {
func bindStartAllCommandFlags(cmd *cobra.Command) {
cmd.Flags().StringP(
"dags", "d", "", "location of DAG files (default is $HOME/.dagu/dags)",
"dags", "d", "", "location of DAG files (default is $HOME/.config/dagu/dags)",
)
cmd.Flags().StringP("host", "s", "", "server host (default is localhost)")
cmd.Flags().StringP("port", "p", "", "server port (default is 8080)")

View File

@ -2,26 +2,28 @@ package cmd
import (
"testing"
"github.com/dagu-dev/dagu/internal/test"
)
func TestStartCommand(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
setup := test.SetupTest(t)
defer setup.Cleanup()
tests := []cmdTest{
{
args: []string{"start", testDAGFile("start.yaml")},
args: []string{"start", testDAGFile("success.yaml")},
expectedOut: []string{"1 finished"},
},
{
args: []string{"start", testDAGFile("start_with_params.yaml")},
args: []string{"start", testDAGFile("params.yaml")},
expectedOut: []string{"params is p1 and p2"},
},
{
args: []string{
"start",
`--params="p3 p4"`,
testDAGFile("start_with_params.yaml"),
testDAGFile("params.yaml"),
},
expectedOut: []string{"params is p3 and p4"},
},

View File

@ -4,14 +4,15 @@ import (
"testing"
"github.com/dagu-dev/dagu/internal/dag/scheduler"
"github.com/dagu-dev/dagu/internal/test"
)
func TestStatusCommand(t *testing.T) {
t.Run("Status command should run", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("StatusDAG", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
dagFile := testDAGFile("status.yaml")
dagFile := testDAGFile("long.yaml")
// Start the DAG.
done := make(chan struct{})
@ -20,7 +21,12 @@ func TestStatusCommand(t *testing.T) {
close(done)
}()
testLastStatusEventual(t, setup.dataStore.HistoryStore(), dagFile, scheduler.StatusRunning)
testLastStatusEventual(
t,
setup.DataStore().HistoryStore(),
dagFile,
scheduler.StatusRunning,
)
// Check the current status.
testRunCommand(t, statusCmd(), cmdTest{

View File

@ -5,14 +5,15 @@ import (
"time"
"github.com/dagu-dev/dagu/internal/dag/scheduler"
"github.com/dagu-dev/dagu/internal/test"
)
func TestStopCommand(t *testing.T) {
t.Run("Stop a DAG", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("StopDAG", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
dagFile := testDAGFile("stop.yaml")
dagFile := testDAGFile("long2.yaml")
// Start the DAG.
done := make(chan struct{})
@ -24,7 +25,12 @@ func TestStopCommand(t *testing.T) {
time.Sleep(time.Millisecond * 100)
// Wait for the DAG running.
testLastStatusEventual(t, setup.dataStore.HistoryStore(), dagFile, scheduler.StatusRunning)
testLastStatusEventual(
t,
setup.DataStore().HistoryStore(),
dagFile,
scheduler.StatusRunning,
)
// Stop the DAG.
testRunCommand(t, stopCmd(), cmdTest{
@ -32,7 +38,9 @@ func TestStopCommand(t *testing.T) {
expectedOut: []string{"Stopping..."}})
// Check the last execution is cancelled.
testLastStatusEventual(t, setup.dataStore.HistoryStore(), dagFile, scheduler.StatusCancel)
testLastStatusEventual(
t, setup.DataStore().HistoryStore(), dagFile, scheduler.StatusCancel,
)
<-done
})
}

3
cmd/testdata/long2.yaml vendored Normal file
View File

@ -0,0 +1,3 @@
steps:
- name: "1"
command: "sleep 1000"

View File

@ -1,3 +0,0 @@
steps:
- name: "1"
command: "sleep 1000"

View File

@ -1,41 +1,36 @@
version: "3.9"
services:
# init container updates permission
init:
image: "ghcr.io/dagu-dev/dagu:latest"
user: root
volumes:
- dagu:/home/dagu/.dagu
command: chown -R dagu /home/dagu/.dagu/
# ui web server process
server:
image: "ghcr.io/dagu-dev/dagu:latest"
environment:
- DAGU_PORT=8080
- DAGU_DAGS=/home/dagu/.dagu/dags
restart: unless-stopped
ports:
- "8080:8080"
volumes:
- dagu:/home/dagu/.dagu
- ./dags/:/home/dagu/.dagu/dags
depends_on:
- init
# scheduler process
scheduler:
image: "ghcr.io/dagu-dev/dagu:latest"
environment:
- DAGU_DAGS=/home/dagu/.dagu/dags
restart: unless-stopped
volumes:
- dagu:/home/dagu/.dagu
- ./dags/:/home/dagu/.dagu/dags
command: dagu scheduler
depends_on:
- init
# init container updates permission
init:
image: "ghcr.io/dagu-dev/dagu:latest"
user: root
volumes:
- dagu_config:/home/dagu/.config/dagu
- dagu_data:/home/dagu/.local/share
command: chown -R dagu /home/dagu/.config/dagu/ /home/dagu/.local/share
# ui server process
server:
image: "ghcr.io/dagu-dev/dagu:latest"
environment:
- DAGU_PORT=8080
restart: unless-stopped
ports:
- "8080:8080"
volumes:
- dagu_config:/home/dagu/.config/dagu
- dagu_data:/home/dagu/.local/share
depends_on:
- init
# scheduler process
scheduler:
image: "ghcr.io/dagu-dev/dagu:latest"
restart: unless-stopped
volumes:
- dagu_config:/home/dagu/.config/dagu
- dagu_data:/home/dagu/.local/share
command: dagu scheduler
depends_on:
- init
volumes:
dagu: {}
dagu_config: {}
dagu_data: {}

View File

@ -3,7 +3,7 @@
Base Configuration for all DAGs
=====================================
Creating a base configuration (default path: ``~/.dagu/config.yaml``) is a convenient way to organize shared settings among all DAGs. The path to the base configuration file can be configured. See :ref:`Configuration Options` for more details.
Creating a base configuration (default path: ``~/.config/dagu/base.yaml``) is a convenient way to organize shared settings among all DAGs. The path to the base configuration file can be configured. See :ref:`Configuration Options` for more details.
Example:

View File

@ -23,7 +23,7 @@ The following environment variables can be used to configure the Dagu. Default v
- ``DAGU_DATA_DIR`` (``$DAGU_HOME/data``): The directory where application data will be stored.
- ``DAGU_SUSPEND_FLAGS_DIR`` (``$DAGU_HOME/suspend``): The directory containing DAG suspend flags.
- ``DAGU_ADMIN_LOG_DIR`` (``$DAGU_HOME/logs/admin``): The directory where admin logs will be stored.
- ``DAGU_BASE_CONFIG`` (``$DAGU_HOME/config.yaml``): The path to the base configuration file.
- ``DAGU_BASE_CONFIG`` (``$DAGU_HOME/base.yaml``): The path to the base configuration file.
- ``DAGU_NAVBAR_COLOR`` (``""``): The color to use for the navigation bar. E.g., ``red`` or ``#ff0000``.
- ``DAGU_NAVBAR_TITLE`` (``Dagu``): The title to display in the navigation bar. E.g., ``Dagu - PROD`` or ``Dagu - DEV``
- ``DAGU_WORK_DIR``: The working directory for DAGs. If not set, the default value is DAG location. Also you can set the working directory for each DAG steps in the DAG configuration file. For more information, see :ref:`specifying working dir`.
@ -62,7 +62,7 @@ You can create ``admin.yaml`` file in the ``$DAGU_HOME`` directory (default: ``$
authToken: <token for API access> # API token
# Base Config
baseConfig: <base DAG config path> # default: ${DAGU_HOME}/config.yaml
baseConfig: <base DAG config path> # default: ${DAGU_HOME}/base.yaml
# Working Directory
workDir: <working directory for DAGs> # default: DAG location

View File

@ -27,13 +27,13 @@ msgstr "すべてのDAGの基本設定"
#: ../../source/base_config.rst:6 0f42fc55990a404f9e436073ba71ba93
#, fuzzy
msgid ""
"Creating a base configuration (default path: ``~/.dagu/config.yaml``) is "
"Creating a base configuration (default path: ``~/.config/dagu/base.yaml``) is "
"a convenient way to organize shared settings among all workflows. The "
"path to the base configuration file can be configured. See "
":ref:`Configuration Options` for more details."
msgstr ""
"基本設定(デフォルトのパス: "
"``~/.dagu/config.yaml``を作成することは、すべてのDAG間で共有設定を整理するのに便利な方法です。基本設定ファイルへのパスは設定可能です。詳細については、:ref:`Configuration"
"``~/.config/dagu/base.yaml``を作成することは、すべてのDAG間で共有設定を整理するのに便利な方法です。基本設定ファイルへのパスは設定可能です。詳細については、:ref:`Configuration"
" Options` を参照してください。"
#: ../../source/base_config.rst:8 b7458496869246558d3efeee58648edd

View File

@ -95,10 +95,10 @@ msgstr ""
#: ../../source/config.rst:26 5198113ba7f54794bbeb0077ec25bdc2
msgid ""
"``DAGU_BASE_CONFIG`` (``$DAGU_HOME/config.yaml``): The path to the base "
"``DAGU_BASE_CONFIG`` (``$DAGU_HOME/base.yaml``): The path to the base "
"configuration file."
msgstr ""
"``DAGU_BASE_CONFIG`` (``$DAGU_HOME/config.yaml``): 基本設定ファイルへのパス。"
"``DAGU_BASE_CONFIG`` (``$DAGU_HOME/base.yaml``): 基本設定ファイルへのパス。"
#: ../../source/config.rst:27 f185a5aa23a049b484202296d52e9272
msgid ""

View File

@ -493,19 +493,19 @@ msgstr "``steps``: DAGで実行するステップのリスト。"
#: ../../source/yaml_format.rst:482 d2ef99deab3945e59788eee68e7d87f5
msgid ""
"In addition, a global configuration file, ``$DAGU_HOME/config.yaml``, can"
"In addition, a global configuration file, ``$DAGU_HOME/base.yaml``, can"
" be used to gather common settings, such as ``logDir`` or ``env``."
msgstr ""
"さらに、グローバル設定ファイル ``$DAGU_HOME/config.yaml`` を使用して、"
"さらに、グローバル設定ファイル ``$DAGU_HOME/base.yaml`` を使用して、"
"``logDir`` や ``env`` などの共通設定を集めることができます。"
#: ../../source/yaml_format.rst:484 9a4859890b2f49c5a66923630d9282e6
msgid ""
"Note: If ``DAGU_HOME`` environment variable is not set, the default path "
"is ``$HOME/.dagu/config.yaml``."
"is ``$HOME/.dagu/base.yaml``."
msgstr ""
"注意: ``DAGU_HOME`` 環境変数が設定されていない場合、デフォルトのパスは"
" ``$HOME/.dagu/config.yaml`` です。"
" ``$HOME/.dagu/base.yaml`` です。"
#: ../../source/yaml_format.rst:522 d8004407254d47db8fda9e238eadc212
msgid "Step"

View File

@ -26,12 +26,12 @@ msgstr "所有工作流的基本配置"
#: ../../source/base_config.rst:6 0f42fc55990a404f9e436073ba71ba93
#, fuzzy
msgid ""
"Creating a base configuration (default path: ``~/.dagu/config.yaml``) is "
"Creating a base configuration (default path: ``~/.config/dagu/base.yaml``) is "
"a convenient way to organize shared settings among all workflows. The "
"path to the base configuration file can be configured. See "
":ref:`Configuration Options` for more details."
msgstr ""
"创建基本配置(默认路径:``~/.dagu/config.yaml``)是组织所有工作流之间共享设置的一种便捷方式。可以配置基本配置文件的路径。有关更多详细信息,请参阅"
"创建基本配置(默认路径:``~/.config/dagu/base.yaml``)是组织所有工作流之间共享设置的一种便捷方式。可以配置基本配置文件的路径。有关更多详细信息,请参阅"
" :ref:`配置选项`。"
#: ../../source/base_config.rst:8 b7458496869246558d3efeee58648edd

View File

@ -86,7 +86,7 @@ msgstr ""
#: ../../source/config.rst:26 2ab3dddd69504f17a31a31ec15cc5299
msgid ""
"``DAGU_BASE_CONFIG`` (``$DAGU_HOME/config.yaml``): The path to the base "
"``DAGU_BASE_CONFIG`` (``$DAGU_HOME/base.yaml``): The path to the base "
"configuration file."
msgstr ""

View File

@ -477,17 +477,17 @@ msgstr "``steps``: 要在 DAG 中执行的步骤列表。"
#: ../../source/yaml_format.rst:482 8993958642c54df8af8f8feb63a61306
msgid ""
"In addition, a global configuration file, ``$DAGU_HOME/config.yaml``, can"
"In addition, a global configuration file, ``$DAGU_HOME/base.yaml``, can"
" be used to gather common settings, such as ``logDir`` or ``env``."
msgstr ""
"此外,可以使用全局配置文件 ``$DAGU_HOME/config.yaml`` 来收集常见设置,例如 ``logDir`` 或 ``env``。"
"此外,可以使用全局配置文件 ``$DAGU_HOME/base.yaml`` 来收集常见设置,例如 ``logDir`` 或 ``env``。"
#: ../../source/yaml_format.rst:484 bee44595f6004cc0b34f4e2bbfdb1217
msgid ""
"Note: If ``DAGU_HOME`` environment variable is not set, the default path "
"is ``$HOME/.dagu/config.yaml``."
"is ``$HOME/.dagu/base.yaml``."
msgstr ""
"注意:如果未设置 ``DAGU_HOME`` 环境变量,默认路径为 ``$HOME/.dagu/config.yaml``。"
"注意:如果未设置 ``DAGU_HOME`` 环境变量,默认路径为 ``$HOME/.dagu/base.yaml``。"
#: ../../source/yaml_format.rst:522 f7dc490d92784e0695de6deee2b84465
msgid "Step"

View File

@ -520,9 +520,9 @@ This section provides a comprehensive list of available fields that can be used
- ``handlerOn``: The command to execute when a DAG or step succeeds, fails, cancels, or exits.
- ``steps``: A list of steps to execute in the DAG.
In addition, a global configuration file, ``$DAGU_HOME/config.yaml``, can be used to gather common settings, such as ``logDir`` or ``env``.
In addition, a global configuration file, ``$DAGU_HOME/base.yaml``, can be used to gather common settings, such as ``logDir`` or ``env``.
Note: If ``DAGU_HOME`` environment variable is not set, the default path is ``$HOME/.dagu/config.yaml``.
Note: If ``DAGU_HOME`` environment variable is not set, the default path is ``$HOME/.dagu/base.yaml``.
Example:

View File

@ -1,9 +0,0 @@
params: "`date +'%Y%m%d'`"
schedule: "*/2 * * * *"
steps:
- name: step 1
command: echo $1
- name: step 2
command: sleep 1
depends:
- step 1

View File

@ -1,41 +0,0 @@
version: "3.9"
services:
# init container updates permission
init:
image: "ghcr.io/dagu-dev/dagu:latest"
user: root
volumes:
- dagu:/home/dagu/.dagu
command: chown -R dagu /home/dagu/.dagu/
# ui server process
server:
image: "ghcr.io/dagu-dev/dagu:latest"
environment:
- DAGU_PORT=8080
- DAGU_DAGS=/home/dagu/.dagu/dags
restart: unless-stopped
ports:
- "8080:8080"
volumes:
- dagu:/home/dagu/.dagu
- ./dags/:/home/dagu/.dagu/dags
depends_on:
- init
# scheduler process
scheduler:
image: "ghcr.io/dagu-dev/dagu:latest"
environment:
- DAGU_DAGS=/home/dagu/.dagu/dags
restart: unless-stopped
volumes:
- dagu:/home/dagu/.dagu
- ./dags/:/home/dagu/.dagu/dags
command: dagu scheduler
depends_on:
- init
volumes:
dagu: {}

6
go.mod
View File

@ -1,8 +1,9 @@
module github.com/dagu-dev/dagu
go 1.22.0
go 1.22.5
require (
github.com/adrg/xdg v0.5.0
github.com/docker/docker v20.10.21+incompatible
github.com/fsnotify/fsnotify v1.7.0
github.com/go-chi/chi/v5 v5.0.8
@ -29,6 +30,7 @@ require (
github.com/spf13/cobra v1.8.0
github.com/spf13/viper v1.18.2
github.com/stretchr/testify v1.9.0
github.com/yohamta/gomerger v0.0.1
go.uber.org/fx v1.20.0
go.uber.org/goleak v1.3.0
golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
@ -261,6 +263,6 @@ require (
github.com/samber/lo v1.38.1
golang.org/x/crypto v0.24.0
golang.org/x/net v0.26.0
golang.org/x/sys v0.21.0
golang.org/x/sys v0.22.0
gopkg.in/yaml.v3 v3.0.1 // indirect
)

8
go.sum
View File

@ -68,6 +68,8 @@ github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2y
github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE=
github.com/OpenPeeDeeP/depguard/v2 v2.2.0 h1:vDfG60vDtIuf0MEOhmLlLLSzqaRM8EMcgJPdp74zmpA=
github.com/OpenPeeDeeP/depguard/v2 v2.2.0/go.mod h1:CIzddKRvLBC4Au5aYP/i3nyaWQ+ClszLIuVocRiCYFQ=
github.com/adrg/xdg v0.5.0 h1:dDaZvhMXatArP1NPHhnfaQUqWBLBsmx1h1HXQdMoFCY=
github.com/adrg/xdg v0.5.0/go.mod h1:dDdY4M4DF9Rjy4kHPeNL+ilVF+p2lK8IdM9/rTSGcI4=
github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk=
github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
github.com/alecthomas/go-check-sumtype v0.1.4 h1:WCvlB3l5Vq5dZQTFmodqL2g68uHiSwwlWcT5a2FGK0c=
@ -699,6 +701,8 @@ github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5Jsjqto
github.com/yeya24/promlinter v0.3.0/go.mod h1:cDfJQQYv9uYciW60QT0eeHlFodotkYZlL+YcPQN+mW4=
github.com/ykadowak/zerologlint v0.1.5 h1:Gy/fMz1dFQN9JZTPjv1hxEk+sRWm05row04Yoolgdiw=
github.com/ykadowak/zerologlint v0.1.5/go.mod h1:KaUskqF3e/v59oPmdq1U1DnKcuHokl2/K1U4pmIELKg=
github.com/yohamta/gomerger v0.0.1 h1:FFKDbEwp3iaFvAGqgkKQLnViX8J953hz7QYhII9h5Fw=
github.com/yohamta/gomerger v0.0.1/go.mod h1:1ABCGxp1LkuJIeTJ2Ik0d2EOK2p9iRTesKWz5X1Tu+E=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@ -929,8 +933,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

View File

@ -7,7 +7,6 @@ import (
"log"
"net/http"
"os"
"path"
"path/filepath"
"regexp"
"sync"
@ -528,13 +527,13 @@ func (a *Agent) setupLog() error {
// It is DAG.LogDir + DAG.Name (with invalid characters replaced with '_').
// It is used to write the stdout and stderr of the steps.
if a.dag.LogDir != "" {
a.logDir = path.Join(a.logDir, util.ValidFilename(a.dag.Name))
a.logDir = filepath.Join(a.logDir, util.ValidFilename(a.dag.Name))
}
absFilepath := filepath.Join(a.logDir, createLogfileName(a.dag.Name, a.reqID, time.Now()))
// Create the log directory
if err := os.MkdirAll(path.Dir(absFilepath), 0755); err != nil {
if err := os.MkdirAll(filepath.Dir(absFilepath), 0755); err != nil {
return err
}

View File

@ -4,78 +4,33 @@ import (
"context"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"syscall"
"testing"
"time"
"github.com/dagu-dev/dagu/internal/agent"
"github.com/dagu-dev/dagu/internal/persistence"
"github.com/dagu-dev/dagu/internal/persistence/client"
"github.com/dagu-dev/dagu/internal/test"
"github.com/dagu-dev/dagu/internal/config"
"github.com/dagu-dev/dagu/internal/dag"
"github.com/dagu-dev/dagu/internal/dag/scheduler"
"github.com/dagu-dev/dagu/internal/engine"
"github.com/dagu-dev/dagu/internal/persistence/model"
"github.com/dagu-dev/dagu/internal/util"
"github.com/stretchr/testify/require"
)
type testSetup struct {
homeDir string
engine engine.Engine
dataStore persistence.DataStores
cfg *config.Config
}
func (t testSetup) cleanup() {
_ = os.RemoveAll(t.homeDir)
}
// setupTest sets temporary directories and loads the configuration.
func setupTest(t *testing.T) testSetup {
t.Helper()
tmpDir := util.MustTempDir("dagu_test")
err := os.Setenv("HOME", tmpDir)
require.NoError(t, err)
cfg, err := config.Load()
require.NoError(t, err)
dataStore := client.NewDataStores(&client.NewDataStoresArgs{
DataDir: path.Join(tmpDir, ".dagu", "data"),
})
eng := engine.New(&engine.NewEngineArgs{
DataStore: dataStore,
WorkDir: cfg.WorkDir,
Executable: cfg.Executable,
})
return testSetup{
homeDir: tmpDir,
engine: eng,
dataStore: dataStore,
cfg: cfg,
}
}
func TestAgent_Run(t *testing.T) {
t.Parallel()
t.Run("Run a DAG successfully", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("RunDAG", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
dg := testLoadDAG(t, "run.yaml")
eng := setup.engine
eng := setup.Engine()
dagAgent := agent.New(&agent.NewAagentArgs{
DAG: dg,
LogDir: setup.cfg.LogDir,
LogDir: setup.Config.LogDir,
Engine: eng,
DataStore: setup.dataStore,
DataStore: setup.DataStore(),
})
latestStatus, err := eng.GetLatestStatus(dg)
@ -95,18 +50,18 @@ func TestAgent_Run(t *testing.T) {
return status.Status == scheduler.StatusSuccess
}, time.Second*2, time.Millisecond*100)
})
t.Run("Old history files are deleted", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("DeleteOldHistory", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
// Create a history file by running a DAG
dg := testLoadDAG(t, "simple.yaml")
eng := setup.engine
eng := setup.Engine()
dagAgent := agent.New(&agent.NewAagentArgs{
DAG: dg,
LogDir: setup.cfg.LogDir,
LogDir: setup.Config.LogDir,
Engine: eng,
DataStore: setup.dataStore,
DataStore: setup.DataStore(),
})
err := dagAgent.Run(context.Background())
require.NoError(t, err)
@ -117,9 +72,9 @@ func TestAgent_Run(t *testing.T) {
dg.HistRetentionDays = 0
dagAgent = agent.New(&agent.NewAagentArgs{
DAG: dg,
LogDir: setup.cfg.LogDir,
LogDir: setup.Config.LogDir,
Engine: eng,
DataStore: setup.dataStore,
DataStore: setup.DataStore(),
})
err = dagAgent.Run(context.Background())
require.NoError(t, err)
@ -128,17 +83,17 @@ func TestAgent_Run(t *testing.T) {
history = eng.GetRecentHistory(dg, 2)
require.Equal(t, 1, len(history))
})
t.Run("It should not run a DAG if it is already running", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("AlreadyRunning", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
dg := testLoadDAG(t, "is_running.yaml")
eng := setup.engine
eng := setup.Engine()
dagAgent := agent.New(&agent.NewAagentArgs{
DAG: dg,
LogDir: setup.cfg.LogDir,
LogDir: setup.Config.LogDir,
Engine: eng,
DataStore: setup.dataStore,
DataStore: setup.DataStore(),
})
go func() {
@ -153,29 +108,29 @@ func TestAgent_Run(t *testing.T) {
dagAgent = agent.New(&agent.NewAagentArgs{
DAG: dg,
LogDir: setup.cfg.LogDir,
LogDir: setup.Config.LogDir,
Engine: eng,
DataStore: setup.dataStore,
DataStore: setup.DataStore(),
})
err := dagAgent.Run(context.Background())
require.Error(t, err)
require.Contains(t, err.Error(), "is already running")
})
t.Run("It should not run a DAG if the precondition is not met", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("PreConditionNotMet", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
dg := testLoadDAG(t, "multiple_steps.yaml")
eng := setup.engine
eng := setup.Engine()
// Precondition is not met
dg.Preconditions = []dag.Condition{{Condition: "`echo 1`", Expected: "0"}}
dagAgent := agent.New(&agent.NewAagentArgs{
DAG: dg,
LogDir: setup.cfg.LogDir,
LogDir: setup.Config.LogDir,
Engine: eng,
DataStore: setup.dataStore,
DataStore: setup.DataStore(),
})
err := dagAgent.Run(context.Background())
require.Error(t, err)
@ -186,16 +141,16 @@ func TestAgent_Run(t *testing.T) {
require.Equal(t, scheduler.NodeStatusNone, status.Nodes[0].Status)
require.Equal(t, scheduler.NodeStatusNone, status.Nodes[1].Status)
})
t.Run("Run a DAG and finish with an error", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("FinishWithError", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
// Run a DAG that fails
dagAgent := agent.New(&agent.NewAagentArgs{
DAG: testLoadDAG(t, "error.yaml"),
LogDir: setup.cfg.LogDir,
Engine: setup.engine,
DataStore: setup.dataStore,
LogDir: setup.Config.LogDir,
Engine: setup.Engine(),
DataStore: setup.DataStore(),
})
err := dagAgent.Run(context.Background())
require.Error(t, err)
@ -203,19 +158,19 @@ func TestAgent_Run(t *testing.T) {
// Check if the status is saved correctly
require.Equal(t, scheduler.StatusError, dagAgent.Status().Status)
})
t.Run("Run a DAG and receive a signal", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("ReceiveSignal", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
abortFunc := func(a *agent.Agent) { a.Signal(syscall.SIGTERM) }
dg := testLoadDAG(t, "sleep.yaml")
eng := setup.engine
eng := setup.Engine()
dagAgent := agent.New(&agent.NewAagentArgs{
DAG: dg,
LogDir: setup.cfg.LogDir,
LogDir: setup.Config.LogDir,
Engine: eng,
DataStore: setup.dataStore,
DataStore: setup.DataStore(),
})
go func() {
@ -238,16 +193,16 @@ func TestAgent_Run(t *testing.T) {
return status.Status == scheduler.StatusCancel
}, time.Second*1, time.Millisecond*100)
})
t.Run("Run a DAG and execute the exit handler", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("ExitHandler", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
dg := testLoadDAG(t, "on_exit.yaml")
dagAgent := agent.New(&agent.NewAagentArgs{
DAG: dg,
LogDir: setup.cfg.LogDir,
Engine: setup.engine,
DataStore: setup.dataStore,
LogDir: setup.Config.LogDir,
Engine: setup.Engine(),
DataStore: setup.DataStore(),
})
err := dagAgent.Run(context.Background())
require.NoError(t, err)
@ -266,18 +221,18 @@ func TestAgent_Run(t *testing.T) {
func TestAgent_DryRun(t *testing.T) {
t.Parallel()
t.Run("Dry-run a DAG successfully", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("DryRun", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
dg := testLoadDAG(t, "dry.yaml")
eng := setup.engine
eng := setup.Engine()
dagAgent := agent.New(&agent.NewAagentArgs{
DAG: dg,
Dry: true,
LogDir: setup.cfg.LogDir,
LogDir: setup.Config.LogDir,
Engine: eng,
DataStore: setup.dataStore,
DataStore: setup.DataStore(),
})
err := dagAgent.Run(context.Background())
@ -295,19 +250,19 @@ func TestAgent_DryRun(t *testing.T) {
func TestAgent_Retry(t *testing.T) {
t.Parallel()
t.Run("Retry a DAG", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("RetryDAG", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
// retry.yaml has a DAG that fails
dg := testLoadDAG(t, "retry.yaml")
eng := setup.engine
eng := setup.Engine()
dagAgent := agent.New(&agent.NewAagentArgs{
DAG: dg,
LogDir: setup.cfg.LogDir,
LogDir: setup.Config.LogDir,
Engine: eng,
DataStore: setup.dataStore,
DataStore: setup.DataStore(),
})
err := dagAgent.Run(context.Background())
require.Error(t, err)
@ -325,9 +280,9 @@ func TestAgent_Retry(t *testing.T) {
dagAgent = agent.New(&agent.NewAagentArgs{
DAG: dg,
RetryTarget: status,
LogDir: setup.cfg.LogDir,
LogDir: setup.Config.LogDir,
Engine: eng,
DataStore: setup.dataStore,
DataStore: setup.DataStore(),
})
err = dagAgent.Run(context.Background())
require.NoError(t, err)
@ -346,18 +301,18 @@ func TestAgent_Retry(t *testing.T) {
func TestAgent_HandleHTTP(t *testing.T) {
t.Parallel()
t.Run("Handle HTTP requests and return the status of the DAG", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("HTTP_Valid", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
// Start a long-running DAG
dg := testLoadDAG(t, "handle_http.yaml")
eng := setup.engine
eng := setup.Engine()
dagAgent := agent.New(&agent.NewAagentArgs{
DAG: dg,
LogDir: setup.cfg.LogDir,
LogDir: setup.Config.LogDir,
Engine: eng,
DataStore: setup.dataStore,
DataStore: setup.DataStore(),
})
go func() {
err := dagAgent.Run(context.Background())
@ -392,18 +347,18 @@ func TestAgent_HandleHTTP(t *testing.T) {
}, time.Second*2, time.Millisecond*100)
})
t.Run("Handle invalid HTTP requests", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("HTTP_InvalidRequest", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
// Start a long-running DAG
dg := testLoadDAG(t, "handle_http2.yaml")
eng := setup.engine
eng := setup.Engine()
dagAgent := agent.New(&agent.NewAagentArgs{
DAG: dg,
LogDir: setup.cfg.LogDir,
LogDir: setup.Config.LogDir,
Engine: eng,
DataStore: setup.dataStore,
DataStore: setup.DataStore(),
})
go func() {
@ -435,18 +390,18 @@ func TestAgent_HandleHTTP(t *testing.T) {
return status.Status == scheduler.StatusCancel
}, time.Second*2, time.Millisecond*100)
})
t.Run("Handle cancel request and stop the DAG", func(t *testing.T) {
setup := setupTest(t)
defer setup.cleanup()
t.Run("HTTP_HandleCancel", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
// Start a long-running DAG
dg := testLoadDAG(t, "handle_http3.yaml")
eng := setup.engine
eng := setup.Engine()
dagAgent := agent.New(&agent.NewAagentArgs{
DAG: dg,
LogDir: setup.cfg.LogDir,
LogDir: setup.Config.LogDir,
Engine: eng,
DataStore: setup.dataStore,
DataStore: setup.DataStore(),
})
go func() {
@ -507,7 +462,7 @@ func (h *mockResponseWriter) WriteHeader(statusCode int) {
// testLoadDAG load the specified DAG file for testing
// without base config or parameters.
func testLoadDAG(t *testing.T, name string) *dag.DAG {
file := path.Join(util.MustGetwd(), "testdata", name)
file := filepath.Join(util.MustGetwd(), "testdata", name)
dg, err := dag.Load("", file, "")
require.NoError(t, err)
return dg

View File

@ -5,7 +5,7 @@ import (
"io"
"log"
"os"
"path"
"path/filepath"
"testing"
"github.com/dagu-dev/dagu/internal/util"
@ -27,7 +27,7 @@ func TestTeeWriter(t *testing.T) {
}()
// Create a temporary file and tee the log to the file.
tmpLogFile := path.Join(util.MustTempDir("test-tee"), "test.log")
tmpLogFile := filepath.Join(util.MustTempDir("test-tee"), "test.log")
logFile, err := os.Create(tmpLogFile)
require.NoError(t, err)

View File

@ -2,15 +2,18 @@ package config
import (
"fmt"
"log"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
"github.com/adrg/xdg"
"github.com/spf13/viper"
)
// Config represents the configuration for the server.
type Config struct {
Host string // Server host
Port int // Server port
@ -41,22 +44,19 @@ type TLS struct {
KeyFile string
}
var lock sync.Mutex
var configLock sync.Mutex
const envPrefix = "DAGU"
func Load() (*Config, error) {
lock.Lock()
defer lock.Unlock()
configLock.Lock()
defer configLock.Unlock()
viper.SetEnvPrefix(envPrefix)
viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
// Bind environment variables with config keys.
bindEnvs()
// Set default values for config keys.
if err := setDefaults(); err != nil {
if err := setupViper(); err != nil {
return nil, err
}
@ -79,138 +79,226 @@ func Load() (*Config, error) {
// Set environment variables specified in the config file.
cfg.Env.Range(func(k, v any) bool {
_ = os.Setenv(k.(string), v.(string))
if err := os.Setenv(k.(string), v.(string)); err != nil {
log.Printf("failed to set env variable %s: %v", k, err)
}
return true
})
return &cfg, nil
}
var defaults = Config{
Host: "127.0.0.1",
Port: 8080,
IsBasicAuth: false,
NavbarTitle: "Dagu",
IsAuthToken: false,
LatestStatusToday: false,
APIBaseURL: "/api/v1",
AdminLogsDir: path.Join(logDir, "admin"),
const (
// Application name.
appName = "dagu"
)
func setupViper() error {
// Bind environment variables with config keys.
bindEnvs()
// Set default values for config keys.
// Directories
baseDirs := getBaseDirs()
viper.SetDefault("dags", baseDirs.dags)
viper.SetDefault("suspendFlagsDir", baseDirs.suspendFlags)
viper.SetDefault("dataDir", baseDirs.data)
viper.SetDefault("logDir", baseDirs.logs)
viper.SetDefault("adminLogsDir", baseDirs.adminLogs)
// Base config file
viper.SetDefault("baseConfig", getBaseConfigPath(baseDirs))
// Other defaults
viper.SetDefault("host", "127.0.0.1")
viper.SetDefault("port", "8080")
viper.SetDefault("navbarTitle", "Dagu")
viper.SetDefault("apiBaseURL", "/api/v1")
// Set executable path
// This is used for invoking the workflow process on the server.
return setExecutableDefault()
}
type baseDirs struct {
config string
dags string
suspendFlags string
data string
logs string
adminLogs string
}
const (
// Constants for config.
appHomeDefault = ".dagu"
legacyAppHome = "DAGU_HOME"
// Default base config file.
baseConfig = "config.yaml"
legacyConfigDir = ".dagu"
legacyConfigDirEnvKey = "DAGU_HOME"
// default directories
dagsDir = "dags"
dataDir = "data"
logDir = "logs"
suspendDir = "suspend"
)
func setDefaults() error {
var (
// ConfigDir is the directory to store DAGs and other configuration files.
ConfigDir = getConfigDir()
// DataDir is the directory to store history data.
DataDir = getDataDir()
// LogsDir is the directory to store logs.
LogsDir = getLogsDir()
)
func getBaseDirs() baseDirs {
return baseDirs{
config: ConfigDir,
dags: filepath.Join(ConfigDir, dagsDir),
suspendFlags: filepath.Join(ConfigDir, suspendDir),
data: DataDir,
logs: LogsDir,
adminLogs: filepath.Join(LogsDir, "admin"),
}
}
func setExecutableDefault() error {
executable, err := os.Executable()
if err != nil {
return fmt.Errorf("failed to get executable path: %w", err)
}
paths := getDefaultPaths()
viper.SetDefault("host", defaults.Host)
viper.SetDefault("port", defaults.Port)
viper.SetDefault("executable", executable)
viper.SetDefault("dags", path.Join(paths.configDir, dagsDir))
viper.SetDefault("workDir", defaults.WorkDir)
viper.SetDefault("isBasicAuth", defaults.IsBasicAuth)
viper.SetDefault("basicAuthUsername", defaults.BasicAuthUsername)
viper.SetDefault("basicAuthPassword", defaults.BasicAuthPassword)
viper.SetDefault("logEncodingCharset", defaults.LogEncodingCharset)
viper.SetDefault("baseConfig", path.Join(paths.configDir, baseConfig))
viper.SetDefault("logDir", path.Join(paths.configDir, logDir))
viper.SetDefault("dataDir", path.Join(paths.configDir, dataDir))
viper.SetDefault("suspendFlagsDir", path.Join(paths.configDir, suspendDir))
viper.SetDefault("adminLogsDir", path.Join(paths.configDir, defaults.AdminLogsDir))
viper.SetDefault("navbarColor", defaults.NavbarColor)
viper.SetDefault("navbarTitle", defaults.NavbarTitle)
viper.SetDefault("isAuthToken", defaults.IsAuthToken)
viper.SetDefault("authToken", defaults.AuthToken)
viper.SetDefault("latestStatusToday", defaults.LatestStatusToday)
viper.SetDefault("apiBaseURL", defaults.APIBaseURL)
return nil
}
func getLogsDir() string {
if v, ok := getLegacyConfigPath(); ok {
// For backward compatibility.
return filepath.Join(v, "logs")
}
return filepath.Join(xdg.DataHome, appName, "logs")
}
func getDataDir() string {
if v, ok := getLegacyConfigPath(); ok {
// For backward compatibility.
return filepath.Join(v, "data")
}
return filepath.Join(xdg.DataHome, appName, "history")
}
func getConfigDir() string {
if v, ok := getLegacyConfigPath(); ok {
return v
}
if v := os.Getenv("XDG_CONFIG_HOME"); v != "" {
return filepath.Join(v, appName)
}
return filepath.Join(getHomeDir(), ".config", appName)
}
func getHomeDir() string {
dir, err := os.UserHomeDir()
if err != nil {
log.Fatalf("could not determine home directory: %v", err)
return ""
}
return dir
}
const (
// Base config file name for all DAGs.
baseConfig = "base.yaml"
// Legacy config path for backward compatibility.
legacyBaseConfig = "base.yaml"
)
func getBaseConfigPath(b baseDirs) string {
legacyPath := filepath.Join(b.config, legacyBaseConfig)
if _, err := os.Stat(legacyPath); err == nil {
return legacyPath
}
return filepath.Join(b.config, baseConfig)
}
func getLegacyConfigPath() (string, bool) {
// For backward compatibility.
// If the environment variable is set, use it.
if v := os.Getenv(legacyConfigDirEnvKey); v != "" {
return v, true
}
// If not, check if the legacyPath config directory exists.
legacyPath := filepath.Join(getHomeDir(), legacyConfigDir)
if _, err := os.Stat(legacyPath); err == nil {
return legacyPath, true
}
return "", false
}
func bindEnvs() {
_ = viper.BindEnv("executable", "DAGU_EXECUTABLE")
_ = viper.BindEnv("dags", "DAGU_DAGS_DIR")
_ = viper.BindEnv("workDir", "DAGU_WORK_DIR")
// Server configurations
_ = viper.BindEnv("logEncodingCharset", "DAGU_LOG_ENCODING_CHARSET")
_ = viper.BindEnv("navbarColor", "DAGU_NAVBAR_COLOR")
_ = viper.BindEnv("navbarTitle", "DAGU_NAVBAR_TITLE")
_ = viper.BindEnv("apiBaseURL", "DAGU_API_BASE_URL")
// Basic authentication
_ = viper.BindEnv("isBasicAuth", "DAGU_IS_BASICAUTH")
_ = viper.BindEnv("basicAuthUsername", "DAGU_BASICAUTH_USERNAME")
_ = viper.BindEnv("basicAuthPassword", "DAGU_BASICAUTH_PASSWORD")
_ = viper.BindEnv("logEncodingCharset", "DAGU_LOG_ENCODING_CHARSET")
// TLS configurations
_ = viper.BindEnv("tls.certFile", "DAGU_CERT_FILE")
_ = viper.BindEnv("tls.keyFile", "DAGU_KEY_FILE")
// Auth Token
_ = viper.BindEnv("isAuthToken", "DAGU_IS_AUTHTOKEN")
_ = viper.BindEnv("authToken", "DAGU_AUTHTOKEN")
// Executables
_ = viper.BindEnv("executable", "DAGU_EXECUTABLE")
// Directories and files
_ = viper.BindEnv("dags", "DAGU_DAGS_DIR")
_ = viper.BindEnv("workDir", "DAGU_WORK_DIR")
_ = viper.BindEnv("baseConfig", "DAGU_BASE_CONFIG")
_ = viper.BindEnv("logDir", "DAGU_LOG_DIR")
_ = viper.BindEnv("dataDir", "DAGU_DATA_DIR")
_ = viper.BindEnv("suspendFlagsDir", "DAGU_SUSPEND_FLAGS_DIR")
_ = viper.BindEnv("adminLogsDir", "DAGU_ADMIN_LOG_DIR")
_ = viper.BindEnv("navbarColor", "DAGU_NAVBAR_COLOR")
_ = viper.BindEnv("navbarTitle", "DAGU_NAVBAR_TITLE")
_ = viper.BindEnv("tls.certFile", "DAGU_CERT_FILE")
_ = viper.BindEnv("tls.keyFile", "DAGU_KEY_FILE")
_ = viper.BindEnv("isAuthToken", "DAGU_IS_AUTHTOKEN")
_ = viper.BindEnv("authToken", "DAGU_AUTHTOKEN")
// Miscellaneous
_ = viper.BindEnv("latestStatusToday", "DAGU_LATEST_STATUS")
_ = viper.BindEnv("apiBaseURL", "DAGU_API_BASE_URL")
}
func loadLegacyEnvs(cfg *Config) {
// For backward compatibility.
// Load old environment variables if they exist.
if v := os.Getenv("DAGU__ADMIN_NAVBAR_COLOR"); v != "" {
log.Println("DAGU__ADMIN_NAVBAR_COLOR is deprecated. Use DAGU_NAVBAR_COLOR instead.")
cfg.NavbarColor = v
}
if v := os.Getenv("DAGU__ADMIN_NAVBAR_TITLE"); v != "" {
log.Println("DAGU__ADMIN_NAVBAR_TITLE is deprecated. Use DAGU_NAVBAR_TITLE instead.")
cfg.NavbarTitle = v
}
if v := os.Getenv("DAGU__ADMIN_PORT"); v != "" {
if i, err := strconv.Atoi(v); err == nil {
log.Println("DAGU__ADMIN_PORT is deprecated. Use DAGU_PORT instead.")
cfg.Port = i
}
}
if v := os.Getenv("DAGU__ADMIN_HOST"); v != "" {
log.Println("DAGU__ADMIN_HOST is deprecated. Use DAGU_HOST instead.")
cfg.Host = v
}
if v := os.Getenv("DAGU__DATA"); v != "" {
log.Println("DAGU__DATA is deprecated. Use DAGU_DATA_DIR instead.")
cfg.DataDir = v
}
if v := os.Getenv("DAGU__SUSPEND_FLAGS_DIR"); v != "" {
log.Println("DAGU__SUSPEND_FLAGS_DIR is deprecated. Use DAGU_SUSPEND_FLAGS_DIR instead.")
cfg.SuspendFlagsDir = v
}
if v := os.Getenv("DAGU__ADMIN_LOGS_DIR"); v != "" {
log.Println("DAGU__ADMIN_LOGS_DIR is deprecated. Use DAGU_ADMIN_LOG_DIR instead.")
cfg.AdminLogsDir = v
}
}
type defaultPaths struct {
configDir string
// Add more paths here if needed.
}
func getDefaultPaths() defaultPaths {
var paths defaultPaths
if appDir := os.Getenv(legacyAppHome); appDir != "" {
paths.configDir = appDir
} else {
home, err := os.UserHomeDir()
if err != nil {
panic(err)
}
paths.configDir = path.Join(home, appHomeDefault)
}
return paths
}

View File

@ -3,11 +3,10 @@ package dag
import (
"fmt"
"os"
"path"
"path/filepath"
"reflect"
"testing"
"github.com/dagu-dev/dagu/internal/config"
"github.com/stretchr/testify/require"
)
@ -403,25 +402,23 @@ schedule:
func TestLoad(t *testing.T) {
// Base config has the following values:
// MailOn: {Failure: true, Success: false}
t.Run("WithBaseConfig", func(t *testing.T) {
cfg, err := config.Load()
require.NoError(t, err)
t.Run("OverrideBaseConfig", func(t *testing.T) {
baseConfig := filepath.Join(testdataDir, "base.yaml")
// Overwrite the base config with the following values:
// MailOn: {Failure: false, Success: false}
dg, err := Load(cfg.BaseConfig, path.Join(testdataDir, "overwrite.yaml"), "")
dg, err := Load(baseConfig, filepath.Join(testdataDir, "overwrite.yaml"), "")
require.NoError(t, err)
// The MailOn key should be overwritten.
require.Equal(t, &MailOn{Failure: false, Success: false}, dg.MailOn)
require.Equal(t, dg.HistRetentionDays, 7)
})
t.Run("WithoutBaseConfig", func(t *testing.T) {
cfg, err := config.Load()
require.NoError(t, err)
t.Run("NoOverrideBaseConfig", func(t *testing.T) {
baseConfig := filepath.Join(testdataDir, "base.yaml")
// no_overwrite.yaml does not have the MailOn key.
dg, err := Load(cfg.BaseConfig, path.Join(testdataDir, "no_overwrite.yaml"), "")
dg, err := Load(baseConfig, filepath.Join(testdataDir, "no_overwrite.yaml"), "")
require.NoError(t, err)
// The MailOn key should be the same as the base config.

View File

@ -4,7 +4,7 @@ import (
// nolint // gosec
"crypto/md5"
"fmt"
"path"
"path/filepath"
"strings"
"time"
@ -177,7 +177,7 @@ func (d *DAG) setup() {
}
// set the default working directory for the steps if not set
dir := path.Dir(d.Location)
dir := filepath.Dir(d.Location)
for i := range d.Steps {
d.Steps[i].setup(dir)
}
@ -214,7 +214,7 @@ func (d *DAG) HasTag(tag string) bool {
// run in parallel.
func (d *DAG) SockAddr() string {
s := strings.ReplaceAll(d.Location, " ", "_")
name := strings.Replace(path.Base(s), path.Ext(path.Base(s)), "", 1)
name := strings.Replace(filepath.Base(s), filepath.Ext(filepath.Base(s)), "", 1)
// nolint // gosec
h := md5.New()
_, _ = h.Write([]byte(s))
@ -226,7 +226,7 @@ func (d *DAG) SockAddr() string {
if len(name) > lengthLimit {
name = name[:lengthLimit-1]
}
return path.Join("/tmp", fmt.Sprintf("@dagu-%s-%x.sock", name, bs))
return filepath.Join("/tmp", fmt.Sprintf("@dagu-%s-%x.sock", name, bs))
}
// String implements the Stringer interface.

View File

@ -1,8 +1,7 @@
package dag
import (
"os"
"path"
"path/filepath"
"testing"
"github.com/dagu-dev/dagu/internal/util"
@ -10,21 +9,12 @@ import (
)
var (
testdataDir = path.Join(util.MustGetwd(), "testdata")
testHomeDir = path.Join(util.MustGetwd(), "testdata/home")
testdataDir = filepath.Join(util.MustGetwd(), "testdata")
)
func TestMain(m *testing.M) {
err := os.Setenv("HOME", testHomeDir)
if err != nil {
panic(err)
}
os.Exit(m.Run())
}
func TestDAG_String(t *testing.T) {
t.Run("DefaltConfig", func(t *testing.T) {
dg, err := Load("", path.Join(testdataDir, "default.yaml"), "")
dg, err := Load("", filepath.Join(testdataDir, "default.yaml"), "")
require.NoError(t, err)
ret := dg.String()

View File

@ -1,11 +1,10 @@
package dag
import (
"path"
"path/filepath"
"testing"
"time"
"github.com/dagu-dev/dagu/internal/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -19,27 +18,27 @@ func Test_Load(t *testing.T) {
}{
{
name: "WithExt",
file: path.Join(testdataDir, "loader_test.yaml"),
expectedLocation: path.Join(testdataDir, "loader_test.yaml"),
file: filepath.Join(testdataDir, "loader_test.yaml"),
expectedLocation: filepath.Join(testdataDir, "loader_test.yaml"),
},
{
name: "WithoutExt",
file: path.Join(testdataDir, "loader_test"),
expectedLocation: path.Join(testdataDir, "loader_test.yaml"),
file: filepath.Join(testdataDir, "loader_test"),
expectedLocation: filepath.Join(testdataDir, "loader_test.yaml"),
},
{
name: "InvalidPath",
file: path.Join(testdataDir, "not_existing_file.yaml"),
file: filepath.Join(testdataDir, "not_existing_file.yaml"),
expectedError: "no such file or directory",
},
{
name: "InvalidDAG",
file: path.Join(testdataDir, "err_decode.yaml"),
file: filepath.Join(testdataDir, "err_decode.yaml"),
expectedError: "has invalid keys: invalidkey",
},
{
name: "InvalidYAML",
file: path.Join(testdataDir, "err_parse.yaml"),
file: filepath.Join(testdataDir, "err_parse.yaml"),
expectedError: "cannot unmarshal",
},
}
@ -59,7 +58,7 @@ func Test_Load(t *testing.T) {
func Test_LoadMetadata(t *testing.T) {
t.Run("Metadata", func(t *testing.T) {
dg, err := LoadMetadata(path.Join(testdataDir, "default.yaml"))
dg, err := LoadMetadata(filepath.Join(testdataDir, "default.yaml"))
require.NoError(t, err)
require.Equal(t, dg.Name, "default")
@ -69,13 +68,8 @@ func Test_LoadMetadata(t *testing.T) {
}
func Test_loadBaseConfig(t *testing.T) {
t.Run("BaseConfigFile", func(t *testing.T) {
// The base config file is set on the global config
// This should be `testdata/home/.dagu/config.yaml`.
cfg, err := config.Load()
require.NoError(t, err)
dg, err := loadBaseConfig(cfg.BaseConfig, buildOpts{})
t.Run("LoadBaseConfigFile", func(t *testing.T) {
dg, err := loadBaseConfig(filepath.Join(testdataDir, "base.yaml"), buildOpts{})
require.NotNil(t, dg)
require.NoError(t, err)
})
@ -83,7 +77,7 @@ func Test_loadBaseConfig(t *testing.T) {
func Test_LoadDefaultConfig(t *testing.T) {
t.Run("DefaultConfigWithoutBaseConfig", func(t *testing.T) {
file := path.Join(testdataDir, "default.yaml")
file := filepath.Join(testdataDir, "default.yaml")
dg, err := Load("", file, "")
require.NoError(t, err)
@ -99,7 +93,7 @@ func Test_LoadDefaultConfig(t *testing.T) {
require.Len(t, dg.Steps, 1)
assert.Equal(t, "1", dg.Steps[0].Name, "1")
assert.Equal(t, "true", dg.Steps[0].Command, "true")
assert.Equal(t, path.Dir(file), dg.Steps[0].Dir)
assert.Equal(t, filepath.Dir(file), dg.Steps[0].Dir)
})
}

View File

@ -5,7 +5,7 @@ import (
"fmt"
"math/rand"
"os"
"path"
"path/filepath"
"syscall"
"testing"
"time"
@ -105,7 +105,7 @@ func TestStdout(t *testing.T) {
runTestNode(t, n)
f := path.Join(os.Getenv("HOME"), n.data.Step.Stdout)
f := filepath.Join(os.Getenv("HOME"), n.data.Step.Stdout)
dat, _ := os.ReadFile(f)
require.Equal(t, "done\n", string(dat))
}
@ -127,11 +127,11 @@ echo Stderr message >&2
runTestNode(t, n)
f := path.Join(os.Getenv("HOME"), n.data.Step.Stderr)
f := filepath.Join(os.Getenv("HOME"), n.data.Step.Stderr)
dat, _ := os.ReadFile(f)
require.Equal(t, "Stderr message\n", string(dat))
f = path.Join(os.Getenv("HOME"), n.data.Step.Stdout)
f = filepath.Join(os.Getenv("HOME"), n.data.Step.Stdout)
dat, _ = os.ReadFile(f)
require.Equal(t, "Stdout message\n", string(dat))
}

View File

@ -3,7 +3,7 @@ package scheduler
import (
"context"
"os"
"path"
"path/filepath"
"sync/atomic"
"syscall"
"testing"
@ -170,7 +170,7 @@ func TestSchedulerCancel(t *testing.T) {
}
func TestSchedulerRetryFail(t *testing.T) {
cmd := path.Join(util.MustGetwd(), "testdata/testfile.sh")
cmd := filepath.Join(util.MustGetwd(), "testdata/testfile.sh")
g, sc, err := testSchedule(t,
dag.Step{
Name: "1",
@ -207,9 +207,9 @@ func TestSchedulerRetryFail(t *testing.T) {
}
func TestSchedulerRetrySuccess(t *testing.T) {
cmd := path.Join(util.MustGetwd(), "testdata/testfile.sh")
cmd := filepath.Join(util.MustGetwd(), "testdata/testfile.sh")
tmpDir, err := os.MkdirTemp("", "scheduler_test")
tmpFile := path.Join(tmpDir, "flag")
tmpFile := filepath.Join(tmpDir, "flag")
require.NoError(t, err)
defer os.Remove(tmpDir)

View File

@ -2,84 +2,29 @@ package engine_test
import (
"net/http"
"os"
"path"
"path/filepath"
"sync"
"testing"
"time"
"github.com/dagu-dev/dagu/internal/persistence"
"github.com/dagu-dev/dagu/internal/persistence/client"
"github.com/dagu-dev/dagu/internal/config"
"github.com/dagu-dev/dagu/internal/dag"
"github.com/dagu-dev/dagu/internal/dag/scheduler"
"github.com/dagu-dev/dagu/internal/engine"
"github.com/dagu-dev/dagu/internal/persistence/model"
"github.com/dagu-dev/dagu/internal/sock"
"github.com/dagu-dev/dagu/internal/test"
"github.com/dagu-dev/dagu/internal/util"
"github.com/stretchr/testify/require"
)
var testdataDir = path.Join(util.MustGetwd(), "./testdata")
var lock sync.Mutex
func setupTest(t *testing.T) (
string, engine.Engine, persistence.DataStores, *config.Config,
) {
t.Helper()
lock.Lock()
defer lock.Unlock()
tmpDir := util.MustTempDir("dagu_test")
_ = os.Setenv("HOME", tmpDir)
cfg, _ := config.Load()
dataStore := client.NewDataStores(&client.NewDataStoresArgs{
DataDir: path.Join(tmpDir, ".dagu", "data"),
DAGs: testdataDir,
})
exec := path.Join(util.MustGetwd(), "../../bin/dagu")
return tmpDir,
engine.New(
&engine.NewEngineArgs{DataStore: dataStore, Executable: exec},
),
dataStore, cfg
}
func setupTestTmpDir(
t *testing.T,
) (string, engine.Engine, persistence.DataStores, *config.Config) {
t.Helper()
tmpDir := util.MustTempDir("dagu_test")
_ = os.Setenv("HOME", tmpDir)
cfg, _ := config.Load()
dataStore := client.NewDataStores(&client.NewDataStoresArgs{
DataDir: path.Join(tmpDir, ".dagu", "data"),
DAGs: path.Join(tmpDir, ".dagu", "dags"),
})
exec := path.Join(util.MustGetwd(), "../../bin/dagu")
return tmpDir,
engine.New(
&engine.NewEngineArgs{DataStore: dataStore, Executable: exec}),
dataStore, cfg
}
var testdataDir = filepath.Join(util.MustGetwd(), "./testdata")
func TestEngine_GetStatus(t *testing.T) {
t.Run("Valid", func(t *testing.T) {
tmpDir, eng, _, _ := setupTest(t)
defer func() {
_ = os.RemoveAll(tmpDir)
}()
file := testDAG("get_status.yaml")
setup := test.SetupTest(t)
defer setup.Cleanup()
file := testDAG("sleep1.yaml")
eng := setup.Engine()
dagStatus, err := eng.GetStatus(file)
require.NoError(t, err)
@ -112,10 +57,10 @@ func TestEngine_GetStatus(t *testing.T) {
require.Equal(t, scheduler.StatusNone, curStatus.Status)
})
t.Run("InvalidDAGName", func(t *testing.T) {
tmpDir, eng, _, _ := setupTest(t)
defer func() {
_ = os.RemoveAll(tmpDir)
}()
setup := test.SetupTest(t)
defer setup.Cleanup()
eng := setup.Engine()
dagStatus, err := eng.GetStatus(testDAG("invalid_dag"))
require.Error(t, err)
@ -125,21 +70,20 @@ func TestEngine_GetStatus(t *testing.T) {
require.Error(t, dagStatus.Error)
})
t.Run("UpdateStatus", func(t *testing.T) {
tmpDir, eng, dataStore, _ := setupTest(t)
defer func() {
_ = os.RemoveAll(tmpDir)
}()
setup := test.SetupTest(t)
defer setup.Cleanup()
var (
file = testDAG("update_status.yaml")
file = testDAG("success.yaml")
requestID = "test-update-status"
now = time.Now()
eng = setup.Engine()
)
dagStatus, err := eng.GetStatus(file)
require.NoError(t, err)
historyStore := dataStore.HistoryStore()
historyStore := setup.DataStore().HistoryStore()
err = historyStore.Open(dagStatus.DAG.Location, now, requestID)
require.NoError(t, err)
@ -170,13 +114,12 @@ func TestEngine_GetStatus(t *testing.T) {
require.Equal(t, newStatus, statusByRequestID.Nodes[0].Status)
})
t.Run("InvalidUpdateStatusWithInvalidReqID", func(t *testing.T) {
tmpDir, eng, _, _ := setupTest(t)
defer func() {
_ = os.RemoveAll(tmpDir)
}()
setup := test.SetupTest(t)
defer setup.Cleanup()
var (
file = testDAG("update_status_failed.yaml")
eng = setup.Engine()
file = testDAG("sleep1.yaml")
wrongReqID = "invalid-request-id"
)
@ -195,31 +138,28 @@ func TestEngine_GetStatus(t *testing.T) {
// nolint // paralleltest
func TestEngine_RunDAG(t *testing.T) {
t.Run("Start", func(t *testing.T) {
tmpDir, eng, _, _ := setupTest(t)
defer func() {
_ = os.RemoveAll(tmpDir)
}()
file := testDAG("start.yaml")
t.Run("RunDAG", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
eng := setup.Engine()
file := testDAG("success.yaml")
dagStatus, err := eng.GetStatus(file)
require.NoError(t, err)
err = eng.Start(dagStatus.DAG, "")
require.Error(t, err)
require.NoError(t, err)
status, err := eng.GetLatestStatus(dagStatus.DAG)
require.NoError(t, err)
require.Equal(t, scheduler.StatusError.String(), status.Status.String())
require.Equal(t, scheduler.StatusSuccess.String(), status.Status.String())
})
t.Run("Stop", func(t *testing.T) {
tmpDir, eng, _, _ := setupTest(t)
defer func() {
_ = os.RemoveAll(tmpDir)
}()
file := testDAG("stop.yaml")
setup := test.SetupTest(t)
defer setup.Cleanup()
eng := setup.Engine()
file := testDAG("sleep10.yaml")
dagStatus, err := eng.GetStatus(file)
require.NoError(t, err)
@ -238,13 +178,11 @@ func TestEngine_RunDAG(t *testing.T) {
}, time.Millisecond*1500, time.Millisecond*100)
})
t.Run("Restart", func(t *testing.T) {
tmpDir, eng, _, _ := setupTest(t)
defer func() {
_ = os.RemoveAll(tmpDir)
}()
file := testDAG("restart.yaml")
setup := test.SetupTest(t)
defer setup.Cleanup()
eng := setup.Engine()
file := testDAG("success.yaml")
dagStatus, err := eng.GetStatus(file)
require.NoError(t, err)
@ -256,12 +194,11 @@ func TestEngine_RunDAG(t *testing.T) {
require.Equal(t, scheduler.StatusSuccess, status.Status)
})
t.Run("Retry", func(t *testing.T) {
tmpDir, eng, _, _ := setupTest(t)
defer func() {
_ = os.RemoveAll(tmpDir)
}()
setup := test.SetupTest(t)
defer setup.Cleanup()
file := testDAG("retry.yaml")
eng := setup.Engine()
file := testDAG("success.yaml")
dagStatus, err := eng.GetStatus(file)
require.NoError(t, err)
@ -297,10 +234,10 @@ func TestEngine_RunDAG(t *testing.T) {
func TestEngine_UpdateDAG(t *testing.T) {
t.Parallel()
t.Run("Update", func(t *testing.T) {
tmpDir, eng, _, _ := setupTestTmpDir(t)
defer func() {
_ = os.RemoveAll(tmpDir)
}()
setup := test.SetupTest(t)
defer setup.Cleanup()
eng := setup.Engine()
// valid DAG
validDAG := `name: test DAG
@ -326,10 +263,10 @@ steps:
require.Equal(t, validDAG, spec)
})
t.Run("Remove", func(t *testing.T) {
tmpDir, eng, _, _ := setupTestTmpDir(t)
defer func() {
_ = os.RemoveAll(tmpDir)
}()
setup := test.SetupTest(t)
defer setup.Cleanup()
eng := setup.Engine()
spec := `name: test DAG
steps:
@ -353,30 +290,30 @@ steps:
require.NoError(t, err)
})
t.Run("Create", func(t *testing.T) {
tmpDir, eng, _, _ := setupTestTmpDir(t)
defer func() {
_ = os.RemoveAll(tmpDir)
}()
setup := test.SetupTest(t)
defer setup.Cleanup()
eng := setup.Engine()
id, err := eng.CreateDAG("test-dag")
require.NoError(t, err)
// Check if the new DAG is actually created.
dg, err := dag.Load("",
path.Join(tmpDir, ".dagu", "dags", id+".yaml"), "")
filepath.Join(setup.Config.DAGs, id+".yaml"), "")
require.NoError(t, err)
require.Equal(t, "test-dag", dg.Name)
})
t.Run("Rename", func(t *testing.T) {
tmpDir, eng, _, _ := setupTestTmpDir(t)
defer func() {
_ = os.RemoveAll(tmpDir)
}()
setup := test.SetupTest(t)
defer setup.Cleanup()
eng := setup.Engine()
// Create a DAG to rename.
id, err := eng.CreateDAG("old_name")
require.NoError(t, err)
_, err = eng.GetStatus(path.Join(tmpDir, ".dagu", "dags", id+".yaml"))
_, err = eng.GetStatus(filepath.Join(setup.Config.DAGs, id+".yaml"))
require.NoError(t, err)
// Rename the file.
@ -384,44 +321,43 @@ steps:
// Check if the file is renamed.
require.NoError(t, err)
require.FileExists(t, path.Join(tmpDir, ".dagu", "dags", id+"_renamed.yaml"))
require.FileExists(t, filepath.Join(setup.Config.DAGs, id+"_renamed.yaml"))
})
}
func TestEngin_ReadHistory(t *testing.T) {
t.Parallel()
t.Run("Read empty history", func(t *testing.T) {
tmpDir, eng, _, _ := setupTest(t)
defer func() {
_ = os.RemoveAll(tmpDir)
}()
func TestEngine_ReadHistory(t *testing.T) {
t.Run("TestEngine_Empty", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
file := testDAG("read_status.yaml")
eng := setup.Engine()
file := testDAG("success.yaml")
_, err := eng.GetStatus(file)
require.NoError(t, err)
})
t.Run("Read all history", func(t *testing.T) {
tmpDir, e, _, _ := setupTest(t)
defer func() {
_ = os.RemoveAll(tmpDir)
}()
t.Run("TestEngine_All", func(t *testing.T) {
setup := test.SetupTest(t)
defer setup.Cleanup()
allDagStatus, _, err := e.GetAllStatus()
require.NoError(t, err)
require.Greater(t, len(allDagStatus), 0)
eng := setup.Engine()
pattern := path.Join(testdataDir, "*.yaml")
matches, err := filepath.Glob(pattern)
// Create a DAG
_, err := eng.CreateDAG("test-dag1")
require.NoError(t, err)
if len(matches) != len(allDagStatus) {
t.Fatalf("unexpected number of dags: %d", len(allDagStatus))
}
_, err = eng.CreateDAG("test-dag2")
require.NoError(t, err)
// Get all statuses.
allDagStatus, _, err := eng.GetAllStatus()
require.NoError(t, err)
require.Equal(t, 2, len(allDagStatus))
})
}
func testDAG(name string) string {
return path.Join(testdataDir, name)
return filepath.Join(testdataDir, name)
}
func testNewStatus(dg *dag.DAG, reqID string, status scheduler.Status,

View File

@ -1,3 +0,0 @@
steps:
- name: "1"
command: "true"

View File

@ -1,3 +0,0 @@
steps:
- name: "1"
command: "true"

View File

@ -1,4 +0,0 @@
params: "a b c"
steps:
- name: "1"
command: "true"

View File

@ -1,3 +0,0 @@
steps:
- name: "1"
command: "true"

View File

@ -1,3 +0,0 @@
steps:
- name: "1"
command: "sleep 1"

View File

@ -56,7 +56,7 @@ func (o *DeleteDagParams) BindRequest(r *http.Request, route *middleware.Matched
return nil
}
// bindDagID binds and validates parameter DagID from path.
// bindDagID binds and validates parameter DagID from filepath.
func (o *DeleteDagParams) bindDagID(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {

View File

@ -86,7 +86,7 @@ func (o *GetDagDetailsParams) BindRequest(r *http.Request, route *middleware.Mat
return nil
}
// bindDagID binds and validates parameter DagID from path.
// bindDagID binds and validates parameter DagID from filepath.
func (o *GetDagDetailsParams) bindDagID(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {

View File

@ -84,7 +84,7 @@ func (o *PostDagActionParams) BindRequest(r *http.Request, route *middleware.Mat
return nil
}
// bindDagID binds and validates parameter DagID from path.
// bindDagID binds and validates parameter DagID from filepath.
func (o *PostDagActionParams) bindDagID(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {

View File

@ -5,7 +5,7 @@ import (
"io"
"log"
"net/http"
"path"
"path/filepath"
"text/template"
"github.com/dagu-dev/dagu/internal/constants"
@ -19,7 +19,7 @@ var (
func (srv *Server) useTemplate(
layout string, name string,
) func(http.ResponseWriter, any) {
files := append(baseTemplates(), path.Join(templatePath, layout))
files := append(baseTemplates(), filepath.Join(templatePath, layout))
tmpl, err := template.New(name).Funcs(
defaultFunctions(srv.funcsConfig)).ParseFS(srv.assets, files...,
)
@ -73,7 +73,7 @@ func baseTemplates() []string {
var templateFiles = []string{"base.gohtml"}
ret := make([]string, 0, len(templateFiles))
for _, t := range templateFiles {
ret = append(ret, path.Join(templatePath, t))
ret = append(ret, filepath.Join(templatePath, t))
}
return ret
}

View File

@ -28,8 +28,11 @@ type Cache[T any] struct {
}
func New[T any](cap int, ttl time.Duration) *Cache[T] {
c := &Cache[T]{capacity: cap, ttl: ttl}
return c
return &Cache[T]{
capacity: cap,
ttl: ttl,
stopCh: make(chan struct{}),
}
}
func (c *Cache[T]) Stop() {
@ -56,6 +59,7 @@ func (c *Cache[T]) evict() {
entry := value.(Entry[T])
if time.Now().After(entry.ExpiresAt) {
c.entries.Delete(key)
c.items.Add(-1)
}
return true
})

View File

@ -2,7 +2,7 @@ package grep
import (
"os"
"path"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
@ -11,7 +11,7 @@ import (
func TestGrep(t *testing.T) {
t.Parallel()
wd, _ := os.Getwd()
dir := path.Join(wd, "/testdata")
dir := filepath.Join(wd, "/testdata")
for _, tc := range []struct {
Name string
File string
@ -22,7 +22,7 @@ func TestGrep(t *testing.T) {
}{
{
Name: "simple",
File: path.Join(dir, "test.txt"),
File: filepath.Join(dir, "test.txt"),
Pattern: "b",
Want: []*Match{
{
@ -33,7 +33,7 @@ func TestGrep(t *testing.T) {
},
{
Name: "regexp",
File: path.Join(dir, "test.txt"),
File: filepath.Join(dir, "test.txt"),
Pattern: "^b.",
Opts: &Options{
IsRegexp: true,
@ -47,7 +47,7 @@ func TestGrep(t *testing.T) {
},
{
Name: "before",
File: path.Join(dir, "test.txt"),
File: filepath.Join(dir, "test.txt"),
Pattern: "b",
Opts: &Options{
Before: 1,
@ -61,7 +61,7 @@ func TestGrep(t *testing.T) {
},
{
Name: "before+after",
File: path.Join(dir, "test.txt"),
File: filepath.Join(dir, "test.txt"),
Pattern: "cc",
Opts: &Options{
Before: 2,
@ -76,7 +76,7 @@ func TestGrep(t *testing.T) {
},
{
Name: "before+after,firstline",
File: path.Join(dir, "test.txt"),
File: filepath.Join(dir, "test.txt"),
Pattern: "aa",
Opts: &Options{
Before: 1,
@ -91,7 +91,7 @@ func TestGrep(t *testing.T) {
},
{
Name: "before+after,lastline",
File: path.Join(dir, "test.txt"),
File: filepath.Join(dir, "test.txt"),
Pattern: "ee",
Opts: &Options{
Before: 1,
@ -106,25 +106,25 @@ func TestGrep(t *testing.T) {
},
{
Name: "no match",
File: path.Join(dir, "test.txt"),
File: filepath.Join(dir, "test.txt"),
Pattern: "no match text",
IsErr: true,
},
{
Name: "no file",
File: path.Join(dir, "dummy.txt"),
File: filepath.Join(dir, "dummy.txt"),
Pattern: "aa",
IsErr: true,
},
{
Name: "no pattern",
File: path.Join(dir, "test.txt"),
File: filepath.Join(dir, "test.txt"),
Pattern: "",
IsErr: true,
},
{
Name: "invalid regexp",
File: path.Join(dir, "test.txt"),
File: filepath.Join(dir, "test.txt"),
Pattern: "(aa",
Opts: &Options{
IsRegexp: true,

View File

@ -10,7 +10,6 @@ import (
"io"
"log"
"os"
"path"
"path/filepath"
"regexp"
"sort"
@ -223,8 +222,8 @@ func (*Store) Compact(_, original string) error {
}
newFile := fmt.Sprintf("%s_c.dat",
strings.TrimSuffix(filepath.Base(original), path.Ext(original)))
f := path.Join(filepath.Dir(original), newFile)
strings.TrimSuffix(filepath.Base(original), filepath.Ext(original)))
f := filepath.Join(filepath.Dir(original), newFile)
w := &writer{target: f}
if err := w.open(); err != nil {
return err
@ -270,12 +269,12 @@ func (s *Store) Rename(oldID, newID string) error {
if err != nil {
return err
}
oldPrefix := path.Base(s.prefixWithDirectory(on))
newPrefix := path.Base(s.prefixWithDirectory(nn))
oldPrefix := filepath.Base(s.prefixWithDirectory(on))
newPrefix := filepath.Base(s.prefixWithDirectory(nn))
for _, m := range matches {
base := path.Base(m)
base := filepath.Base(m)
f := strings.Replace(base, oldPrefix, newPrefix, 1)
_ = os.Rename(m, path.Join(newDir, f))
_ = os.Rename(m, filepath.Join(newDir, f))
}
if files, _ := os.ReadDir(oldDir); len(files) == 0 {
_ = os.Remove(oldDir)
@ -434,5 +433,5 @@ func readLineFrom(f *os.File, offset int64) ([]byte, error) {
}
func prefix(dagFile string) string {
return strings.TrimSuffix(filepath.Base(dagFile), path.Ext(dagFile))
return strings.TrimSuffix(filepath.Base(dagFile), filepath.Ext(dagFile))
}

View File

@ -4,7 +4,6 @@ import (
"fmt"
"io"
"os"
"path"
"path/filepath"
"strings"
"testing"
@ -36,7 +35,7 @@ func TestNewDataFile(t *testing.T) {
reqID := "request-id-1"
f, err := db.newFile(d.Location, timestamp, reqID)
require.NoError(t, err)
p := util.ValidFilename(strings.TrimSuffix(path.Base(d.Location), path.Ext(d.Location)))
p := util.ValidFilename(strings.TrimSuffix(filepath.Base(d.Location), filepath.Ext(d.Location)))
require.Regexp(t, fmt.Sprintf("%s.*/%s.20220101.00:00:00.000.%s.dat", p, p, reqID[:8]), f)
_, err = db.newFile("", timestamp, reqID)

View File

@ -3,7 +3,7 @@ package jsondb
import (
"bufio"
"os"
"path"
"path/filepath"
"strings"
"sync"
@ -24,7 +24,7 @@ type writer struct {
// Open opens the writer.
func (w *writer) open() (err error) {
_ = os.MkdirAll(path.Dir(w.target), 0755)
_ = os.MkdirAll(filepath.Dir(w.target), 0755)
w.file, err = util.OpenOrCreateFile(w.target)
if err == nil {
w.writer = bufio.NewWriter(w.file)

View File

@ -79,9 +79,9 @@ func (er *entryReaderImpl) Read(now time.Time) ([]*entry, error) {
if er.engine.IsSuspended(dg.Name) {
continue
}
addEntriesFn(dg, dg.Schedule, Start)
addEntriesFn(dg, dg.StopSchedule, Stop)
addEntriesFn(dg, dg.RestartSchedule, Restart)
addEntriesFn(dg, dg.Schedule, entryTypeStart)
addEntriesFn(dg, dg.StopSchedule, entryTypeStop)
addEntriesFn(dg, dg.RestartSchedule, entryTypeRestart)
}
return entries, nil

View File

@ -2,7 +2,7 @@ package scheduler
import (
"os"
"path"
"path/filepath"
"testing"
"time"
@ -25,7 +25,7 @@ func TestReadEntries(t *testing.T) {
now := time.Date(2020, 1, 1, 1, 0, 0, 0, time.UTC).Add(-time.Second)
entryReader := newEntryReader(newEntryReaderArgs{
DagsDir: path.Join(testdataDir, "invalid_directory"),
DagsDir: filepath.Join(testdataDir, "invalid_directory"),
JobCreator: &mockJobFactory{},
Logger: logger.NewSlogLogger(),
Engine: eng,
@ -73,7 +73,7 @@ func TestReadEntries(t *testing.T) {
})
}
var testdataDir = path.Join(util.MustGetwd(), "testdata")
var testdataDir = filepath.Join(util.MustGetwd(), "testdata")
func setupTest(t *testing.T) (string, engine.Engine) {
t.Helper()
@ -84,7 +84,7 @@ func setupTest(t *testing.T) (string, engine.Engine) {
require.NoError(t, err)
cfg := &config.Config{
DataDir: path.Join(tmpDir, ".dagu", "data"),
DataDir: filepath.Join(tmpDir, ".dagu", "data"),
DAGs: testdataDir,
SuspendFlagsDir: tmpDir,
}

View File

@ -42,7 +42,7 @@ func LifetimeHooks(lc fx.Lifecycle, a *Scheduler) {
lc.Append(
fx.Hook{
OnStart: func(ctx context.Context) (err error) {
return a.Start()
return a.Start(ctx)
},
OnStop: func(_ context.Context) error {
a.Stop()

View File

@ -1,10 +1,11 @@
package scheduler
import (
"context"
"fmt"
"os"
"os/signal"
"path"
"path/filepath"
"sort"
"sync"
"sync/atomic"
@ -47,45 +48,45 @@ type job interface {
type entryType int
const (
Start entryType = iota
Stop
Restart
entryTypeStart entryType = iota
entryTypeStop
entryTypeRestart
)
func (e entryType) String() string {
switch e {
case entryTypeStart:
return "start"
case entryTypeStop:
return "stop"
case entryTypeRestart:
return "restart"
default:
return "unknown"
}
}
func (e *entry) Invoke() error {
if e.Job == nil {
return nil
}
logMsg := fmt.Sprintf("%s job", e.EntryType)
e.Logger.Info(logMsg,
"job", e.Job.String(),
"time", e.Next.Format(time.RFC3339),
)
switch e.EntryType {
case Start:
e.Logger.Info(
"start job",
"job",
e.Job.String(),
"time",
e.Next.Format("2006-01-02 15:04:05"),
)
case entryTypeStart:
return e.Job.Start()
case Stop:
e.Logger.Info(
"stop job",
"job",
e.Job.String(),
"time",
e.Next.Format("2006-01-02 15:04:05"),
)
case entryTypeStop:
return e.Job.Stop()
case Restart:
e.Logger.Info(
"restart job",
"job",
e.Job.String(),
"time",
e.Next.Format("2006-01-02 15:04:05"),
)
case entryTypeRestart:
return e.Job.Restart()
default:
return fmt.Errorf("unknown entry type: %v", e.EntryType)
}
return nil
}
type newSchedulerArgs struct {
@ -103,7 +104,7 @@ func newScheduler(args newSchedulerArgs) *Scheduler {
}
}
func (s *Scheduler) Start() error {
func (s *Scheduler) Start(ctx context.Context) error {
if err := s.setupLogFile(); err != nil {
return fmt.Errorf("setup log file: %w", err)
}
@ -124,6 +125,8 @@ func (s *Scheduler) Start() error {
return
case <-sig:
s.Stop()
case <-ctx.Done():
s.Stop()
}
}()
@ -133,28 +136,33 @@ func (s *Scheduler) Start() error {
return nil
}
func (s *Scheduler) setupLogFile() (err error) {
filename := path.Join(s.logDir, "scheduler.log")
dir := path.Dir(filename)
func (s *Scheduler) setupLogFile() error {
filename := filepath.Join(s.logDir, "scheduler.log")
dir := filepath.Dir(filename)
if err := os.MkdirAll(dir, 0755); err != nil {
return err
return fmt.Errorf("create log directory: %w", err)
}
s.logger.Info("setup log", "filename", filename)
return err
s.logger.Info("Setup log", "filename", filename)
return nil
}
func (s *Scheduler) start() {
t := now().Truncate(time.Second * 60)
// TODO: refactor this to use a ticker
t := now().Truncate(time.Minute)
timer := time.NewTimer(0)
s.running.Store(true)
for {
select {
case <-timer.C:
s.run(t)
t = s.nextTick(t)
timer = time.NewTimer(t.Sub(now()))
case <-s.stop:
_ = timer.Stop()
timer.Reset(t.Sub(now()))
case <-s.stop:
if !timer.Stop() {
<-timer.C
}
return
}
}
@ -194,28 +202,29 @@ func (s *Scheduler) Stop() {
return
}
if s.stop != nil {
s.stop <- struct{}{}
close(s.stop)
}
s.running.Store(false)
s.logger.Info("Scheduler stopped")
}
var (
fixedTime time.Time
lock sync.RWMutex
timeLock sync.RWMutex
)
// setFixedTime sets the fixed time.
// This is used for testing.
func setFixedTime(t time.Time) {
lock.Lock()
defer lock.Unlock()
timeLock.Lock()
defer timeLock.Unlock()
fixedTime = t
}
// now returns the current time.
func now() time.Time {
lock.RLock()
defer lock.RUnlock()
timeLock.RLock()
defer timeLock.RUnlock()
if fixedTime.IsZero() {
return time.Now()
}

View File

@ -1,6 +1,7 @@
package scheduler
import (
"context"
"testing"
"time"
@ -37,7 +38,7 @@ func TestScheduler(t *testing.T) {
})
go func() {
_ = schedulerInstance.Start()
_ = schedulerInstance.Start(context.Background())
}()
time.Sleep(time.Second + time.Millisecond*100)
@ -53,7 +54,7 @@ func TestScheduler(t *testing.T) {
entryReader := &mockEntryReader{
Entries: []*entry{
{
EntryType: Restart,
EntryType: entryTypeRestart,
Job: &mockJob{},
Next: now,
Logger: logger.NewSlogLogger(),
@ -68,7 +69,7 @@ func TestScheduler(t *testing.T) {
})
go func() {
_ = schedulerInstance.Start()
_ = schedulerInstance.Start(context.Background())
}()
defer schedulerInstance.Stop()

107
internal/test/setup.go Normal file
View File

@ -0,0 +1,107 @@
package test
import (
"os"
"path/filepath"
"sync"
"testing"
"github.com/dagu-dev/dagu/internal/config"
"github.com/dagu-dev/dagu/internal/engine"
"github.com/dagu-dev/dagu/internal/persistence"
"github.com/dagu-dev/dagu/internal/persistence/client"
"github.com/dagu-dev/dagu/internal/util"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
type Setup struct {
Config *config.Config
homeDir string
}
func (t Setup) Cleanup() {
_ = os.RemoveAll(t.homeDir)
}
func (t Setup) DataStore() persistence.DataStores {
return client.NewDataStores(&client.NewDataStoresArgs{
DAGs: t.Config.DAGs,
DataDir: t.Config.DataDir,
SuspendFlagsDir: t.Config.SuspendFlagsDir,
LatestStatusToday: t.Config.LatestStatusToday,
})
}
func (t Setup) Engine() engine.Engine {
return engine.New(&engine.NewEngineArgs{
DataStore: t.DataStore(),
Executable: t.Config.Executable,
WorkDir: t.Config.WorkDir,
})
}
var (
lock sync.Mutex
)
func SetupTest(t *testing.T) Setup {
lock.Lock()
defer lock.Unlock()
tmpDir := util.MustTempDir("dagu_test")
err := os.Setenv("HOME", tmpDir)
require.NoError(t, err)
viper.AddConfigPath(config.ConfigDir)
viper.SetConfigType("yaml")
viper.SetConfigName("admin")
config.ConfigDir = filepath.Join(tmpDir, "config")
config.DataDir = filepath.Join(tmpDir, "data")
config.LogsDir = filepath.Join(tmpDir, "log")
cfg, err := config.Load()
require.NoError(t, err)
// Set the executable path to the test binary.
cfg.Executable = filepath.Join(util.MustGetwd(), "../../bin/dagu")
// Set environment variables.
// This is required for some tests that run the executable
_ = os.Setenv("DAGU_DAGS_DIR", cfg.DAGs)
_ = os.Setenv("DAGU_WORK_DIR", cfg.WorkDir)
_ = os.Setenv("DAGU_BASE_CONFIG", cfg.BaseConfig)
_ = os.Setenv("DAGU_LOG_DIR", cfg.LogDir)
_ = os.Setenv("DAGU_DATA_DIR", cfg.DataDir)
_ = os.Setenv("DAGU_SUSPEND_FLAGS_DIR", cfg.SuspendFlagsDir)
_ = os.Setenv("DAGU_ADMIN_LOG_DIR", cfg.AdminLogsDir)
return Setup{
Config: cfg,
homeDir: tmpDir,
}
}
func SetupForDir(t *testing.T, dir string) Setup {
lock.Lock()
defer lock.Unlock()
tmpDir := util.MustTempDir("dagu_test")
err := os.Setenv("HOME", tmpDir)
require.NoError(t, err)
viper.AddConfigPath(dir)
viper.SetConfigType("yaml")
viper.SetConfigName("admin")
cfg, err := config.Load()
require.NoError(t, err)
return Setup{
Config: cfg,
homeDir: tmpDir,
}
}

View File

@ -6,7 +6,7 @@ import (
"io"
"log"
"os"
"path"
"path/filepath"
"testing"
"time"
@ -136,7 +136,7 @@ func Test_OpenOrCreateFile(t *testing.T) {
tmp, err := os.MkdirTemp("", "open_or_create")
require.NoError(t, err)
name := path.Join(tmp, "/file.txt")
name := filepath.Join(tmp, "/file.txt")
f, err := util.OpenOrCreateFile(name)
require.NoError(t, err)
@ -155,7 +155,7 @@ func Test_OpenOrCreateFile(t *testing.T) {
_ = os.RemoveAll(dir)
}()
filename := path.Join(dir, "test.txt")
filename := filepath.Join(dir, "test.txt")
createdFile, err := util.OpenOrCreateFile(filename)
require.NoError(t, err)
defer func() {

View File

@ -9,5 +9,6 @@ import (
_ "github.com/go-swagger/go-swagger/cmd/swagger"
_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
_ "github.com/segmentio/golines"
_ "github.com/yohamta/gomerger"
_ "gotest.tools/gotestsum"
)

View File

@ -1,4 +1,12 @@
import { DAG, DAGStatus, Node, NodeStatus, Schedule, SchedulerStatus, StatusFile } from './index';
import {
DAG,
DAGStatus,
Node,
NodeStatus,
Schedule,
SchedulerStatus,
StatusFile,
} from './index';
export type GetDAGResponse = {
Title: string;
@ -47,8 +55,8 @@ export type GridData = {
};
export type ListWorkflowsResponse = {
DAGs: WorkflowListItem[];
Errors: string[];
DAGs?: WorkflowListItem[];
Errors?: string[];
HasError: boolean;
};
@ -81,4 +89,4 @@ export type WorkflowStatus = {
FinishedAt: string;
Log: string;
Params: string;
};
};

View File

@ -32,7 +32,7 @@ function DAGs() {
const merged = React.useMemo(() => {
const ret: DAGItem[] = [];
if (data) {
if (data && data.DAGs) {
for (const val of data.DAGs) {
if (!val.ErrorT) {
ret.push({
@ -72,8 +72,8 @@ function DAGs() {
{data && (
<React.Fragment>
<DAGErrors
DAGs={data.DAGs}
errors={data.Errors}
DAGs={data.DAGs || []}
errors={data.Errors || []}
hasError={data.HasError}
></DAGErrors>
<DAGTable

View File

@ -31,7 +31,7 @@ function Dashboard() {
return;
}
const m = { ...defaultMetrics };
data.DAGs.forEach((wf) => {
data.DAGs?.forEach((wf) => {
if (wf.Status && wf.Status.Status) {
const status = wf.Status.Status;
m[status] += 1;