From 1910e01da1f237d92c641d8a760deb45a0f1b860 Mon Sep 17 00:00:00 2001 From: yann Date: Fri, 5 Sep 2025 16:54:41 +0200 Subject: [PATCH] first commit --- .dockerignore | 4 + .editorconfig | 14 + .gitattributes | 1 + .github/CODE_OF_CONDUCT.md | 52 + .github/CONTRIBUTING.md | 59 + .github/ISSUE_TEMPLATE/bug-report.md | 27 + .github/ISSUE_TEMPLATE/config.yml | 7 + .github/ISSUE_TEMPLATE/feature-request.md | 22 + .github/SECURITY.md | 17 + .github/releases.md | 41 + .github/workflows/release.yml | 158 ++ .github/workflows/test.yml | 75 + .gitignore | 49 + .golangci.yml | 17 + .mkdocs.yml | 84 ++ .version | 1 + COPYING | 674 +++++++++ Dockerfile | 32 + HACKING.md | 130 ++ README.md | 25 + build.sh | 201 +++ cmd/README.md | 11 + cmd/maddy-pam-helper/README.md | 65 + cmd/maddy-pam-helper/maddy.conf | 3 + cmd/maddy-pam-helper/main.c | 51 + cmd/maddy-pam-helper/main.go | 38 + cmd/maddy-pam-helper/pam.c | 100 ++ cmd/maddy-pam-helper/pam.h | 27 + cmd/maddy-shadow-helper/README.md | 47 + cmd/maddy-shadow-helper/main.go | 71 + cmd/maddy/main.go | 29 + config.go | 120 ++ contrib/README.md | 6 + contrib/kubernetes/chart/.helmignore | 23 + contrib/kubernetes/chart/Chart.yaml | 23 + contrib/kubernetes/chart/README.md | 69 + contrib/kubernetes/chart/files/aliases | 1 + contrib/kubernetes/chart/files/maddy.conf | 171 +++ contrib/kubernetes/chart/templates/NOTES.txt | 0 .../kubernetes/chart/templates/_helpers.tpl | 62 + .../kubernetes/chart/templates/configmap.yaml | 10 + .../chart/templates/deployment.yaml | 113 ++ contrib/kubernetes/chart/templates/pvc.yaml | 22 + .../kubernetes/chart/templates/service.yaml | 27 + .../chart/templates/serviceaccount.yaml | 12 + .../templates/tests/test-connection.yaml | 15 + contrib/kubernetes/chart/values.yaml | 74 + directories.go | 46 + directories_docker.go | 11 + dist/README.md | 41 + dist/apparmor/dev.foxcpp.maddy | 38 + dist/fail2ban/filter.d/maddy-auth.conf | 6 + .../filter.d/maddy-dictonary-attack.conf | 7 + dist/fail2ban/jail.d/maddy-auth.conf | 5 + .../jail.d/maddy-dictonary-attack.conf | 7 + dist/install.sh | 24 + dist/logrotate.d/maddy | 7 + dist/systemd/maddy.service | 82 + dist/systemd/maddy@.service | 78 + dist/vim/ftdetect/maddy-conf.vim | 1 + dist/vim/ftplugin/maddy-conf.vim | 8 + dist/vim/syntax/maddy-conf.vim | 225 +++ docs/docker.md | 81 + docs/faq.md | 119 ++ docs/index.md | 22 + docs/internals/quirks.md | 23 + docs/internals/specifications.md | 291 ++++ docs/internals/sqlite.md | 49 + docs/internals/unicode.md | 96 ++ docs/man/.gitignore | 1 + docs/man/README.md | 16 + docs/man/maddy.1.scd | 41 + docs/man/prepare_md.py | 57 + docs/multiple-domains.md | 157 ++ docs/reference/auth/dovecot_sasl.md | 26 + docs/reference/auth/external.md | 52 + docs/reference/auth/ldap.md | 130 ++ docs/reference/auth/netauth.md | 48 + docs/reference/auth/pam.md | 48 + docs/reference/auth/pass_table.md | 44 + docs/reference/auth/plain_separate.md | 45 + docs/reference/auth/shadow.md | 40 + docs/reference/blob/fs.md | 23 + docs/reference/blob/s3.md | 98 ++ docs/reference/checks/actions.md | 21 + docs/reference/checks/authorize_sender.md | 132 ++ docs/reference/checks/command.md | 96 ++ docs/reference/checks/dkim.md | 63 + docs/reference/checks/dnsbl.md | 173 +++ docs/reference/checks/milter.md | 49 + docs/reference/checks/misc.md | 48 + docs/reference/checks/rspamd.md | 97 ++ docs/reference/checks/spf.md | 97 ++ docs/reference/config-syntax.md | 200 +++ docs/reference/endpoints/imap.md | 164 ++ docs/reference/endpoints/openmetrics.md | 41 + docs/reference/endpoints/smtp.md | 312 ++++ docs/reference/global-config.md | 153 ++ docs/reference/modifiers/dkim.md | 225 +++ docs/reference/modifiers/envelope.md | 63 + docs/reference/modules.md | 76 + docs/reference/smtp-pipeline.md | 408 +++++ docs/reference/storage/imap-filters.md | 70 + docs/reference/storage/imapsql.md | 208 +++ docs/reference/table/auth.md | 6 + docs/reference/table/chain.md | 41 + docs/reference/table/email_localpart.md | 20 + docs/reference/table/email_with_domain.md | 37 + docs/reference/table/file.md | 58 + docs/reference/table/regexp.md | 63 + docs/reference/table/sql_query.md | 120 ++ docs/reference/table/static.md | 21 + docs/reference/targets/queue.md | 95 ++ docs/reference/targets/remote.md | 295 ++++ docs/reference/targets/smtp.md | 123 ++ docs/reference/tls-acme.md | 290 ++++ docs/reference/tls.md | 155 ++ docs/seclevels.md | 89 ++ docs/third-party/dovecot.md | 87 ++ docs/third-party/mailman3.md | 84 ++ docs/third-party/rspamd.md | 38 + docs/third-party/smtp-servers.md | 58 + docs/tutorials/alias-to-remote.md | 123 ++ docs/tutorials/building-from-source.md | 55 + docs/tutorials/pam.md | 94 ++ docs/tutorials/setting-up.md | 259 ++++ docs/upgrading.md | 117 ++ framework/address/doc.go | 21 + framework/address/norm.go | 169 +++ framework/address/norm_test.go | 88 ++ framework/address/rfc6531.go | 85 ++ framework/address/rfc6531_test.go | 41 + framework/address/split.go | 132 ++ framework/address/split_test.go | 110 ++ framework/address/validation.go | 138 ++ framework/address/validation_test.go | 33 + framework/buffer/buffer.go | 60 + framework/buffer/bytesreader.go | 61 + framework/buffer/file.go | 85 ++ framework/buffer/memory.go | 50 + framework/cfgparser/env.go | 67 + framework/cfgparser/imports.go | 176 +++ framework/cfgparser/parse.go | 391 +++++ framework/cfgparser/parse_test.go | 622 ++++++++ framework/config/config.go | 36 + framework/config/directories.go | 47 + framework/config/endpoint.go | 142 ++ framework/config/endpoint_test.go | 55 + framework/config/lexer/LICENSE.APACHE | 201 +++ framework/config/lexer/README.md | 20 + framework/config/lexer/dispenser.go | 264 ++++ framework/config/lexer/dispenser_test.go | 324 ++++ framework/config/lexer/lexer.go | 178 +++ framework/config/lexer/lexer_test.go | 206 +++ framework/config/lexer/parse.go | 42 + framework/config/map.go | 743 +++++++++ framework/config/map_test.go | 495 ++++++ framework/config/module/check_action.go | 184 +++ framework/config/module/interfaces.go | 98 ++ framework/config/module/modconfig.go | 163 ++ framework/config/tls/client.go | 88 ++ framework/config/tls/general.go | 136 ++ framework/config/tls/server.go | 124 ++ framework/dns/debugflags.go | 36 + framework/dns/dnssec.go | 403 +++++ framework/dns/dnssec_test.go | 196 +++ framework/dns/idna.go | 37 + framework/dns/norm.go | 72 + framework/dns/override.go | 53 + framework/dns/resolver.go | 61 + framework/exterrors/dns.go | 35 + framework/exterrors/exterrors.go | 22 + framework/exterrors/fields.go | 74 + framework/exterrors/smtp.go | 146 ++ framework/exterrors/temporary.go | 74 + framework/future/future.go | 105 ++ framework/future/future_test.go | 75 + framework/hooks/hooks.go | 80 + framework/log/log.go | 237 +++ framework/log/orderedjson.go | 85 ++ framework/log/output.go | 74 + framework/log/syslog.go | 62 + framework/log/syslog_stub.go | 37 + framework/log/writer.go | 95 ++ framework/log/zap.go | 57 + framework/logparser/parse.go | 124 ++ framework/logparser/parse_test.go | 113 ++ framework/module/auth.go | 44 + framework/module/blob_store.go | 46 + framework/module/check.go | 113 ++ framework/module/delivery_target.go | 93 ++ framework/module/dummy.go | 87 ++ framework/module/imap_filter.go | 43 + framework/module/instances.go | 105 ++ framework/module/modifier.go | 83 + framework/module/module.go | 90 ++ framework/module/module_specific_data.go | 63 + framework/module/msgmetadata.go | 158 ++ framework/module/mxauth.go | 156 ++ framework/module/partial_delivery.go | 58 + framework/module/registry.go | 103 ++ framework/module/storage.go | 50 + framework/module/table.go | 43 + framework/module/tls_loader.go | 41 + go.mod | 173 +++ go.sum | 1329 +++++++++++++++++ internal/README.md | 24 + internal/auth/auth.go | 53 + internal/auth/auth_test.go | 92 ++ internal/auth/dovecot_sasl/dovecot_sasl.go | 160 ++ internal/auth/external/externalauth.go | 103 ++ internal/auth/external/helperauth.go | 55 + internal/auth/ldap/ldap.go | 289 ++++ internal/auth/netauth/netauth.go | 117 ++ internal/auth/pam/module.go | 94 ++ internal/auth/pam/pam.c | 102 ++ internal/auth/pam/pam.go | 56 + internal/auth/pam/pam.h | 27 + internal/auth/pam/pam_stub.go | 34 + internal/auth/pass_table/hash.go | 180 +++ internal/auth/pass_table/table.go | 198 +++ internal/auth/pass_table/table_test.go | 64 + .../auth/plain_separate/plain_separate.go | 145 ++ .../plain_separate/plain_separate_test.go | 155 ++ internal/auth/sasl.go | 207 +++ internal/auth/sasl_test.go | 89 ++ internal/auth/sasllogin/sasllogin.go | 54 + internal/auth/shadow/module.go | 138 ++ internal/auth/shadow/read.go | 100 ++ internal/auth/shadow/shadow.go | 64 + internal/auth/shadow/verify.go | 74 + internal/authz/lookup.go | 44 + internal/authz/normalization.go | 37 + .../authorize_sender/authorize_sender.go | 305 ++++ internal/check/command/command.go | 401 +++++ internal/check/dkim/dkim.go | 272 ++++ internal/check/dkim/dkim_test.go | 364 +++++ internal/check/dns/dns.go | 163 ++ internal/check/dns/dns_test.go | 116 ++ internal/check/dnsbl/common.go | 197 +++ internal/check/dnsbl/common_test.go | 238 +++ internal/check/dnsbl/dnsbl.go | 435 ++++++ internal/check/dnsbl/dnsbl_test.go | 213 +++ internal/check/milter/milter.go | 446 ++++++ internal/check/milter/milter_test.go | 61 + internal/check/requiretls/requiretls.go | 45 + internal/check/rspamd/rspamd.go | 367 +++++ internal/check/skeleton.go | 101 ++ internal/check/spf/spf.go | 420 ++++++ internal/check/stateless_check.go | 202 +++ internal/cli/app.go | 113 ++ internal/cli/clitools/clitools.go | 118 ++ internal/cli/clitools/termios.go | 82 + internal/cli/clitools/termios_stub.go | 49 + internal/cli/ctl/appendlimit.go | 79 + internal/cli/ctl/hash.go | 139 ++ internal/cli/ctl/imap.go | 893 +++++++++++ internal/cli/ctl/imapacct.go | 301 ++++ internal/cli/ctl/moduleinit.go | 133 ++ internal/cli/ctl/users.go | 246 +++ internal/cli/extflag.go | 60 + internal/dmarc/dmarc.go | 42 + internal/dmarc/evaluate.go | 256 ++++ internal/dmarc/evaluate_test.go | 514 +++++++ internal/dmarc/verifier.go | 174 +++ internal/dmarc/verifier_test.go | 219 +++ internal/dsn/dsn.go | 298 ++++ .../endpoint/dovecot_sasld/dovecot_sasl.go | 127 ++ internal/endpoint/dovecot_sasld/mech_info.go | 33 + internal/endpoint/imap/imap.go | 322 ++++ internal/endpoint/openmetrics/om.go | 107 ++ internal/endpoint/smtp/date.go | 66 + internal/endpoint/smtp/metrics.go | 87 ++ internal/endpoint/smtp/session.go | 656 ++++++++ internal/endpoint/smtp/smtp.go | 429 ++++++ internal/endpoint/smtp/smtp_test.go | 591 ++++++++ internal/endpoint/smtp/smtputf8_test.go | 361 +++++ internal/endpoint/smtp/submission.go | 148 ++ internal/endpoint/smtp/submission_test.go | 165 ++ internal/imap_filter/command/command.go | 207 +++ internal/imap_filter/group.go | 92 ++ internal/libdns/acmedns.go | 28 + internal/libdns/alidns.go | 26 + internal/libdns/cloudflare.go | 25 + internal/libdns/digitalocean.go | 25 + internal/libdns/gandi.go | 38 + internal/libdns/gcore.go | 34 + internal/libdns/googleclouddns.go | 26 + internal/libdns/hetzner.go | 25 + internal/libdns/leaseweb.go | 25 + internal/libdns/metaname.go | 28 + internal/libdns/namecheap.go | 28 + internal/libdns/namedotcom.go | 28 + internal/libdns/provider_module.go | 35 + internal/libdns/rfc2136.go | 28 + internal/libdns/route53.go | 26 + internal/libdns/vultr.go | 25 + internal/limits/limiters/bucket.go | 153 ++ internal/limits/limiters/concurrency.go | 68 + internal/limits/limiters/limiters.go | 35 + internal/limits/limiters/multilimit.go | 69 + internal/limits/limiters/rate.go | 117 ++ internal/limits/limits.go | 234 +++ internal/modify/dkim/dkim.go | 375 +++++ internal/modify/dkim/dkim_test.go | 245 +++ internal/modify/dkim/keys.go | 184 +++ internal/modify/dkim/keys_test.go | 156 ++ internal/modify/group.go | 140 ++ internal/modify/replace_addr.go | 156 ++ internal/modify/replace_addr_test.go | 114 ++ internal/msgpipeline/bench_test.go | 98 ++ internal/msgpipeline/bodynonatomic_test.go | 148 ++ internal/msgpipeline/check_group.go | 67 + internal/msgpipeline/check_runner.go | 349 +++++ internal/msgpipeline/check_test.go | 448 ++++++ internal/msgpipeline/config.go | 397 +++++ internal/msgpipeline/config_test.go | 440 ++++++ internal/msgpipeline/dmarc_test.go | 224 +++ internal/msgpipeline/metrics.go | 47 + internal/msgpipeline/modifier_test.go | 709 +++++++++ internal/msgpipeline/module.go | 70 + internal/msgpipeline/msgpipeline.go | 655 ++++++++ internal/msgpipeline/msgpipeline_test.go | 706 +++++++++ internal/msgpipeline/objname.go | 46 + internal/msgpipeline/regress_test.go | 137 ++ internal/proxy_protocol/proxy_protocol.go | 86 ++ internal/smtpconn/pool/pool.go | 211 +++ internal/smtpconn/smtpconn.go | 558 +++++++ internal/smtpconn/smtpconn_test.go | 41 + internal/smtpconn/smtputf8_test.go | 165 ++ internal/storage/blob/fs/fs.go | 95 ++ internal/storage/blob/fs/fs_test.go | 19 + internal/storage/blob/s3/s3.go | 196 +++ internal/storage/blob/s3/s3_test.go | 71 + internal/storage/blob/test_blob.go | 62 + internal/storage/blob/test_blob_nosqlite.go | 14 + internal/storage/imapsql/bench_test.go | 90 ++ internal/storage/imapsql/delivery.go | 168 +++ .../storage/imapsql/external_blob_store.go | 68 + internal/storage/imapsql/imapsql.go | 438 ++++++ internal/storage/imapsql/maddyctl.go | 42 + internal/storage/imapsql/modernc_sqlite3.go | 26 + internal/storage/imapsql/no_sqlite3.go | 24 + internal/storage/imapsql/sqlite3.go | 26 + internal/table/chain.go | 131 ++ internal/table/email_localpart.go | 70 + internal/table/email_with_domain.go | 89 ++ internal/table/file.go | 258 ++++ internal/table/file_test.go | 222 +++ internal/table/identity.go | 58 + internal/table/regexp.go | 127 ++ internal/table/sql_query.go | 251 ++++ internal/table/sql_query_test.go | 96 ++ internal/table/sql_table.go | 173 +++ internal/table/sqlite3.go | 24 + internal/table/static.go | 77 + internal/target/delivery.go | 34 + internal/target/queue/metrics.go | 35 + internal/target/queue/queue.go | 998 +++++++++++++ internal/target/queue/queue_test.go | 828 ++++++++++ internal/target/queue/timewheel.go | 146 ++ internal/target/queue/timewheel_test.go | 127 ++ internal/target/received.go | 107 ++ internal/target/remote/connect.go | 390 +++++ internal/target/remote/dane.go | 158 ++ internal/target/remote/dane_delivery_test.go | 477 ++++++ internal/target/remote/dane_test.go | 227 +++ internal/target/remote/debugflags.go | 35 + internal/target/remote/metrics.go | 46 + internal/target/remote/mxauth_test.go | 642 ++++++++ internal/target/remote/policy_group.go | 105 ++ internal/target/remote/remote.go | 484 ++++++ internal/target/remote/remote_test.go | 1054 +++++++++++++ internal/target/remote/security.go | 642 ++++++++ internal/target/skeleton.go | 130 ++ internal/target/smtp/sasl.go | 77 + internal/target/smtp/sasl_test.go | 154 ++ internal/target/smtp/smtp_downstream.go | 336 +++++ internal/target/smtp/smtp_downstream_test.go | 272 ++++ internal/target/smtp/smtputf8_test.go | 61 + internal/testutils/bench_delivery.go | 141 ++ internal/testutils/buffer.go | 84 ++ internal/testutils/check.go | 111 ++ internal/testutils/filesystem.go | 34 + internal/testutils/logger.go | 59 + internal/testutils/modifier.go | 122 ++ internal/testutils/multitable.go | 35 + internal/testutils/smtp_server.go | 454 ++++++ internal/testutils/table.go | 31 + internal/testutils/target.go | 344 +++++ internal/tls/acme/acme.go | 164 ++ internal/tls/file.go | 168 +++ internal/tls/self_signed.go | 112 ++ internal/updatepipe/backend.go | 45 + internal/updatepipe/pubsub/pq.go | 86 ++ internal/updatepipe/pubsub/pubsub.go | 11 + internal/updatepipe/pubsub_pipe.go | 101 ++ internal/updatepipe/serialize.go | 69 + internal/updatepipe/unix_pipe.go | 129 ++ internal/updatepipe/update_pipe.go | 62 + maddy.conf | 181 +++ maddy.conf.docker | 182 +++ maddy.go | 434 ++++++ maddy_debug.go | 30 + signal.go | 66 + signal_nonposix.go | 45 + systemd.go | 133 ++ systemd_nonlinux.go | 34 + tests/README.md | 17 + tests/basic_test.go | 63 + tests/build_cover.sh | 5 + tests/conn.go | 289 ++++ tests/cover_test.go | 91 ++ tests/dovecot_sasl_test.go | 206 +++ tests/dovecot_sasld_test.go | 202 +++ tests/gocovcat.go | 92 ++ tests/golangci-noisy.yml | 23 + tests/imap_test.go | 96 ++ tests/imapsql_test.go | 259 ++++ tests/issue327_test.go | 91 ++ tests/limits_test.go | 107 ++ tests/lmtp_test.go | 181 +++ tests/mta_test.go | 135 ++ tests/multiple_domains_test.go | 340 +++++ tests/replace_addr_test.go | 245 +++ tests/run.sh | 17 + tests/smtp_autobuffer_test.go | 347 +++++ tests/smtp_test.go | 667 +++++++++ tests/stress_test.go | 411 +++++ tests/t.go | 484 ++++++ tests/testdata/check_command.sh | 11 + .../testdata/testing+addHeader@maddy.test.hdr | 1 + tests/testdata/testing+reject@maddy.test.exit | 1 + 433 files changed, 61737 insertions(+) create mode 100644 .dockerignore create mode 100644 .editorconfig create mode 100644 .gitattributes create mode 100644 .github/CODE_OF_CONDUCT.md create mode 100644 .github/CONTRIBUTING.md create mode 100644 .github/ISSUE_TEMPLATE/bug-report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature-request.md create mode 100644 .github/SECURITY.md create mode 100644 .github/releases.md create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 .mkdocs.yml create mode 100644 .version create mode 100644 COPYING create mode 100644 Dockerfile create mode 100644 HACKING.md create mode 100644 README.md create mode 100755 build.sh create mode 100644 cmd/README.md create mode 100644 cmd/maddy-pam-helper/README.md create mode 100644 cmd/maddy-pam-helper/maddy.conf create mode 100644 cmd/maddy-pam-helper/main.c create mode 100644 cmd/maddy-pam-helper/main.go create mode 100644 cmd/maddy-pam-helper/pam.c create mode 100644 cmd/maddy-pam-helper/pam.h create mode 100644 cmd/maddy-shadow-helper/README.md create mode 100644 cmd/maddy-shadow-helper/main.go create mode 100644 cmd/maddy/main.go create mode 100644 config.go create mode 100644 contrib/README.md create mode 100644 contrib/kubernetes/chart/.helmignore create mode 100644 contrib/kubernetes/chart/Chart.yaml create mode 100644 contrib/kubernetes/chart/README.md create mode 100644 contrib/kubernetes/chart/files/aliases create mode 100644 contrib/kubernetes/chart/files/maddy.conf create mode 100644 contrib/kubernetes/chart/templates/NOTES.txt create mode 100644 contrib/kubernetes/chart/templates/_helpers.tpl create mode 100644 contrib/kubernetes/chart/templates/configmap.yaml create mode 100644 contrib/kubernetes/chart/templates/deployment.yaml create mode 100644 contrib/kubernetes/chart/templates/pvc.yaml create mode 100644 contrib/kubernetes/chart/templates/service.yaml create mode 100644 contrib/kubernetes/chart/templates/serviceaccount.yaml create mode 100644 contrib/kubernetes/chart/templates/tests/test-connection.yaml create mode 100644 contrib/kubernetes/chart/values.yaml create mode 100644 directories.go create mode 100644 directories_docker.go create mode 100644 dist/README.md create mode 100644 dist/apparmor/dev.foxcpp.maddy create mode 100644 dist/fail2ban/filter.d/maddy-auth.conf create mode 100644 dist/fail2ban/filter.d/maddy-dictonary-attack.conf create mode 100644 dist/fail2ban/jail.d/maddy-auth.conf create mode 100644 dist/fail2ban/jail.d/maddy-dictonary-attack.conf create mode 100755 dist/install.sh create mode 100644 dist/logrotate.d/maddy create mode 100644 dist/systemd/maddy.service create mode 100644 dist/systemd/maddy@.service create mode 100644 dist/vim/ftdetect/maddy-conf.vim create mode 100644 dist/vim/ftplugin/maddy-conf.vim create mode 100644 dist/vim/syntax/maddy-conf.vim create mode 100644 docs/docker.md create mode 100644 docs/faq.md create mode 100644 docs/index.md create mode 100644 docs/internals/quirks.md create mode 100644 docs/internals/specifications.md create mode 100644 docs/internals/sqlite.md create mode 100644 docs/internals/unicode.md create mode 100644 docs/man/.gitignore create mode 100644 docs/man/README.md create mode 100644 docs/man/maddy.1.scd create mode 100644 docs/man/prepare_md.py create mode 100644 docs/multiple-domains.md create mode 100644 docs/reference/auth/dovecot_sasl.md create mode 100644 docs/reference/auth/external.md create mode 100644 docs/reference/auth/ldap.md create mode 100644 docs/reference/auth/netauth.md create mode 100644 docs/reference/auth/pam.md create mode 100644 docs/reference/auth/pass_table.md create mode 100644 docs/reference/auth/plain_separate.md create mode 100644 docs/reference/auth/shadow.md create mode 100644 docs/reference/blob/fs.md create mode 100644 docs/reference/blob/s3.md create mode 100644 docs/reference/checks/actions.md create mode 100644 docs/reference/checks/authorize_sender.md create mode 100644 docs/reference/checks/command.md create mode 100644 docs/reference/checks/dkim.md create mode 100644 docs/reference/checks/dnsbl.md create mode 100644 docs/reference/checks/milter.md create mode 100644 docs/reference/checks/misc.md create mode 100644 docs/reference/checks/rspamd.md create mode 100644 docs/reference/checks/spf.md create mode 100644 docs/reference/config-syntax.md create mode 100644 docs/reference/endpoints/imap.md create mode 100644 docs/reference/endpoints/openmetrics.md create mode 100644 docs/reference/endpoints/smtp.md create mode 100644 docs/reference/global-config.md create mode 100644 docs/reference/modifiers/dkim.md create mode 100644 docs/reference/modifiers/envelope.md create mode 100644 docs/reference/modules.md create mode 100644 docs/reference/smtp-pipeline.md create mode 100644 docs/reference/storage/imap-filters.md create mode 100644 docs/reference/storage/imapsql.md create mode 100644 docs/reference/table/auth.md create mode 100644 docs/reference/table/chain.md create mode 100644 docs/reference/table/email_localpart.md create mode 100644 docs/reference/table/email_with_domain.md create mode 100644 docs/reference/table/file.md create mode 100644 docs/reference/table/regexp.md create mode 100644 docs/reference/table/sql_query.md create mode 100644 docs/reference/table/static.md create mode 100644 docs/reference/targets/queue.md create mode 100644 docs/reference/targets/remote.md create mode 100644 docs/reference/targets/smtp.md create mode 100644 docs/reference/tls-acme.md create mode 100644 docs/reference/tls.md create mode 100644 docs/seclevels.md create mode 100644 docs/third-party/dovecot.md create mode 100644 docs/third-party/mailman3.md create mode 100644 docs/third-party/rspamd.md create mode 100644 docs/third-party/smtp-servers.md create mode 100644 docs/tutorials/alias-to-remote.md create mode 100644 docs/tutorials/building-from-source.md create mode 100644 docs/tutorials/pam.md create mode 100644 docs/tutorials/setting-up.md create mode 100644 docs/upgrading.md create mode 100644 framework/address/doc.go create mode 100644 framework/address/norm.go create mode 100644 framework/address/norm_test.go create mode 100644 framework/address/rfc6531.go create mode 100644 framework/address/rfc6531_test.go create mode 100644 framework/address/split.go create mode 100644 framework/address/split_test.go create mode 100644 framework/address/validation.go create mode 100644 framework/address/validation_test.go create mode 100644 framework/buffer/buffer.go create mode 100644 framework/buffer/bytesreader.go create mode 100644 framework/buffer/file.go create mode 100644 framework/buffer/memory.go create mode 100644 framework/cfgparser/env.go create mode 100644 framework/cfgparser/imports.go create mode 100644 framework/cfgparser/parse.go create mode 100644 framework/cfgparser/parse_test.go create mode 100644 framework/config/config.go create mode 100644 framework/config/directories.go create mode 100644 framework/config/endpoint.go create mode 100644 framework/config/endpoint_test.go create mode 100644 framework/config/lexer/LICENSE.APACHE create mode 100644 framework/config/lexer/README.md create mode 100644 framework/config/lexer/dispenser.go create mode 100644 framework/config/lexer/dispenser_test.go create mode 100644 framework/config/lexer/lexer.go create mode 100644 framework/config/lexer/lexer_test.go create mode 100644 framework/config/lexer/parse.go create mode 100644 framework/config/map.go create mode 100644 framework/config/map_test.go create mode 100644 framework/config/module/check_action.go create mode 100644 framework/config/module/interfaces.go create mode 100644 framework/config/module/modconfig.go create mode 100644 framework/config/tls/client.go create mode 100644 framework/config/tls/general.go create mode 100644 framework/config/tls/server.go create mode 100644 framework/dns/debugflags.go create mode 100644 framework/dns/dnssec.go create mode 100644 framework/dns/dnssec_test.go create mode 100644 framework/dns/idna.go create mode 100644 framework/dns/norm.go create mode 100644 framework/dns/override.go create mode 100644 framework/dns/resolver.go create mode 100644 framework/exterrors/dns.go create mode 100644 framework/exterrors/exterrors.go create mode 100644 framework/exterrors/fields.go create mode 100644 framework/exterrors/smtp.go create mode 100644 framework/exterrors/temporary.go create mode 100644 framework/future/future.go create mode 100644 framework/future/future_test.go create mode 100644 framework/hooks/hooks.go create mode 100644 framework/log/log.go create mode 100644 framework/log/orderedjson.go create mode 100644 framework/log/output.go create mode 100644 framework/log/syslog.go create mode 100644 framework/log/syslog_stub.go create mode 100644 framework/log/writer.go create mode 100644 framework/log/zap.go create mode 100644 framework/logparser/parse.go create mode 100644 framework/logparser/parse_test.go create mode 100644 framework/module/auth.go create mode 100644 framework/module/blob_store.go create mode 100644 framework/module/check.go create mode 100644 framework/module/delivery_target.go create mode 100644 framework/module/dummy.go create mode 100644 framework/module/imap_filter.go create mode 100644 framework/module/instances.go create mode 100644 framework/module/modifier.go create mode 100644 framework/module/module.go create mode 100644 framework/module/module_specific_data.go create mode 100644 framework/module/msgmetadata.go create mode 100644 framework/module/mxauth.go create mode 100644 framework/module/partial_delivery.go create mode 100644 framework/module/registry.go create mode 100644 framework/module/storage.go create mode 100644 framework/module/table.go create mode 100644 framework/module/tls_loader.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/README.md create mode 100644 internal/auth/auth.go create mode 100644 internal/auth/auth_test.go create mode 100644 internal/auth/dovecot_sasl/dovecot_sasl.go create mode 100644 internal/auth/external/externalauth.go create mode 100644 internal/auth/external/helperauth.go create mode 100644 internal/auth/ldap/ldap.go create mode 100644 internal/auth/netauth/netauth.go create mode 100644 internal/auth/pam/module.go create mode 100644 internal/auth/pam/pam.c create mode 100644 internal/auth/pam/pam.go create mode 100644 internal/auth/pam/pam.h create mode 100644 internal/auth/pam/pam_stub.go create mode 100644 internal/auth/pass_table/hash.go create mode 100644 internal/auth/pass_table/table.go create mode 100644 internal/auth/pass_table/table_test.go create mode 100644 internal/auth/plain_separate/plain_separate.go create mode 100644 internal/auth/plain_separate/plain_separate_test.go create mode 100644 internal/auth/sasl.go create mode 100644 internal/auth/sasl_test.go create mode 100644 internal/auth/sasllogin/sasllogin.go create mode 100644 internal/auth/shadow/module.go create mode 100644 internal/auth/shadow/read.go create mode 100644 internal/auth/shadow/shadow.go create mode 100644 internal/auth/shadow/verify.go create mode 100644 internal/authz/lookup.go create mode 100644 internal/authz/normalization.go create mode 100644 internal/check/authorize_sender/authorize_sender.go create mode 100644 internal/check/command/command.go create mode 100644 internal/check/dkim/dkim.go create mode 100644 internal/check/dkim/dkim_test.go create mode 100644 internal/check/dns/dns.go create mode 100644 internal/check/dns/dns_test.go create mode 100644 internal/check/dnsbl/common.go create mode 100644 internal/check/dnsbl/common_test.go create mode 100644 internal/check/dnsbl/dnsbl.go create mode 100644 internal/check/dnsbl/dnsbl_test.go create mode 100644 internal/check/milter/milter.go create mode 100644 internal/check/milter/milter_test.go create mode 100644 internal/check/requiretls/requiretls.go create mode 100644 internal/check/rspamd/rspamd.go create mode 100644 internal/check/skeleton.go create mode 100644 internal/check/spf/spf.go create mode 100644 internal/check/stateless_check.go create mode 100644 internal/cli/app.go create mode 100644 internal/cli/clitools/clitools.go create mode 100644 internal/cli/clitools/termios.go create mode 100644 internal/cli/clitools/termios_stub.go create mode 100644 internal/cli/ctl/appendlimit.go create mode 100644 internal/cli/ctl/hash.go create mode 100644 internal/cli/ctl/imap.go create mode 100644 internal/cli/ctl/imapacct.go create mode 100644 internal/cli/ctl/moduleinit.go create mode 100644 internal/cli/ctl/users.go create mode 100644 internal/cli/extflag.go create mode 100644 internal/dmarc/dmarc.go create mode 100644 internal/dmarc/evaluate.go create mode 100644 internal/dmarc/evaluate_test.go create mode 100644 internal/dmarc/verifier.go create mode 100644 internal/dmarc/verifier_test.go create mode 100644 internal/dsn/dsn.go create mode 100644 internal/endpoint/dovecot_sasld/dovecot_sasl.go create mode 100644 internal/endpoint/dovecot_sasld/mech_info.go create mode 100644 internal/endpoint/imap/imap.go create mode 100644 internal/endpoint/openmetrics/om.go create mode 100644 internal/endpoint/smtp/date.go create mode 100644 internal/endpoint/smtp/metrics.go create mode 100644 internal/endpoint/smtp/session.go create mode 100644 internal/endpoint/smtp/smtp.go create mode 100644 internal/endpoint/smtp/smtp_test.go create mode 100644 internal/endpoint/smtp/smtputf8_test.go create mode 100644 internal/endpoint/smtp/submission.go create mode 100644 internal/endpoint/smtp/submission_test.go create mode 100644 internal/imap_filter/command/command.go create mode 100644 internal/imap_filter/group.go create mode 100644 internal/libdns/acmedns.go create mode 100644 internal/libdns/alidns.go create mode 100644 internal/libdns/cloudflare.go create mode 100644 internal/libdns/digitalocean.go create mode 100644 internal/libdns/gandi.go create mode 100644 internal/libdns/gcore.go create mode 100644 internal/libdns/googleclouddns.go create mode 100644 internal/libdns/hetzner.go create mode 100644 internal/libdns/leaseweb.go create mode 100644 internal/libdns/metaname.go create mode 100644 internal/libdns/namecheap.go create mode 100644 internal/libdns/namedotcom.go create mode 100644 internal/libdns/provider_module.go create mode 100644 internal/libdns/rfc2136.go create mode 100644 internal/libdns/route53.go create mode 100644 internal/libdns/vultr.go create mode 100644 internal/limits/limiters/bucket.go create mode 100644 internal/limits/limiters/concurrency.go create mode 100644 internal/limits/limiters/limiters.go create mode 100644 internal/limits/limiters/multilimit.go create mode 100644 internal/limits/limiters/rate.go create mode 100644 internal/limits/limits.go create mode 100644 internal/modify/dkim/dkim.go create mode 100644 internal/modify/dkim/dkim_test.go create mode 100644 internal/modify/dkim/keys.go create mode 100644 internal/modify/dkim/keys_test.go create mode 100644 internal/modify/group.go create mode 100644 internal/modify/replace_addr.go create mode 100644 internal/modify/replace_addr_test.go create mode 100644 internal/msgpipeline/bench_test.go create mode 100644 internal/msgpipeline/bodynonatomic_test.go create mode 100644 internal/msgpipeline/check_group.go create mode 100644 internal/msgpipeline/check_runner.go create mode 100644 internal/msgpipeline/check_test.go create mode 100644 internal/msgpipeline/config.go create mode 100644 internal/msgpipeline/config_test.go create mode 100644 internal/msgpipeline/dmarc_test.go create mode 100644 internal/msgpipeline/metrics.go create mode 100644 internal/msgpipeline/modifier_test.go create mode 100644 internal/msgpipeline/module.go create mode 100644 internal/msgpipeline/msgpipeline.go create mode 100644 internal/msgpipeline/msgpipeline_test.go create mode 100644 internal/msgpipeline/objname.go create mode 100644 internal/msgpipeline/regress_test.go create mode 100644 internal/proxy_protocol/proxy_protocol.go create mode 100644 internal/smtpconn/pool/pool.go create mode 100644 internal/smtpconn/smtpconn.go create mode 100644 internal/smtpconn/smtpconn_test.go create mode 100644 internal/smtpconn/smtputf8_test.go create mode 100644 internal/storage/blob/fs/fs.go create mode 100644 internal/storage/blob/fs/fs_test.go create mode 100644 internal/storage/blob/s3/s3.go create mode 100644 internal/storage/blob/s3/s3_test.go create mode 100644 internal/storage/blob/test_blob.go create mode 100644 internal/storage/blob/test_blob_nosqlite.go create mode 100644 internal/storage/imapsql/bench_test.go create mode 100644 internal/storage/imapsql/delivery.go create mode 100644 internal/storage/imapsql/external_blob_store.go create mode 100644 internal/storage/imapsql/imapsql.go create mode 100644 internal/storage/imapsql/maddyctl.go create mode 100644 internal/storage/imapsql/modernc_sqlite3.go create mode 100644 internal/storage/imapsql/no_sqlite3.go create mode 100644 internal/storage/imapsql/sqlite3.go create mode 100644 internal/table/chain.go create mode 100644 internal/table/email_localpart.go create mode 100644 internal/table/email_with_domain.go create mode 100644 internal/table/file.go create mode 100644 internal/table/file_test.go create mode 100644 internal/table/identity.go create mode 100644 internal/table/regexp.go create mode 100644 internal/table/sql_query.go create mode 100644 internal/table/sql_query_test.go create mode 100644 internal/table/sql_table.go create mode 100644 internal/table/sqlite3.go create mode 100644 internal/table/static.go create mode 100644 internal/target/delivery.go create mode 100644 internal/target/queue/metrics.go create mode 100644 internal/target/queue/queue.go create mode 100644 internal/target/queue/queue_test.go create mode 100644 internal/target/queue/timewheel.go create mode 100644 internal/target/queue/timewheel_test.go create mode 100644 internal/target/received.go create mode 100644 internal/target/remote/connect.go create mode 100644 internal/target/remote/dane.go create mode 100644 internal/target/remote/dane_delivery_test.go create mode 100644 internal/target/remote/dane_test.go create mode 100644 internal/target/remote/debugflags.go create mode 100644 internal/target/remote/metrics.go create mode 100644 internal/target/remote/mxauth_test.go create mode 100644 internal/target/remote/policy_group.go create mode 100644 internal/target/remote/remote.go create mode 100644 internal/target/remote/remote_test.go create mode 100644 internal/target/remote/security.go create mode 100644 internal/target/skeleton.go create mode 100644 internal/target/smtp/sasl.go create mode 100644 internal/target/smtp/sasl_test.go create mode 100644 internal/target/smtp/smtp_downstream.go create mode 100644 internal/target/smtp/smtp_downstream_test.go create mode 100644 internal/target/smtp/smtputf8_test.go create mode 100644 internal/testutils/bench_delivery.go create mode 100644 internal/testutils/buffer.go create mode 100644 internal/testutils/check.go create mode 100644 internal/testutils/filesystem.go create mode 100644 internal/testutils/logger.go create mode 100644 internal/testutils/modifier.go create mode 100644 internal/testutils/multitable.go create mode 100644 internal/testutils/smtp_server.go create mode 100644 internal/testutils/table.go create mode 100644 internal/testutils/target.go create mode 100644 internal/tls/acme/acme.go create mode 100644 internal/tls/file.go create mode 100644 internal/tls/self_signed.go create mode 100644 internal/updatepipe/backend.go create mode 100644 internal/updatepipe/pubsub/pq.go create mode 100644 internal/updatepipe/pubsub/pubsub.go create mode 100644 internal/updatepipe/pubsub_pipe.go create mode 100644 internal/updatepipe/serialize.go create mode 100644 internal/updatepipe/unix_pipe.go create mode 100644 internal/updatepipe/update_pipe.go create mode 100644 maddy.conf create mode 100644 maddy.conf.docker create mode 100644 maddy.go create mode 100644 maddy_debug.go create mode 100644 signal.go create mode 100644 signal_nonposix.go create mode 100644 systemd.go create mode 100644 systemd_nonlinux.go create mode 100644 tests/README.md create mode 100644 tests/basic_test.go create mode 100755 tests/build_cover.sh create mode 100644 tests/conn.go create mode 100644 tests/cover_test.go create mode 100644 tests/dovecot_sasl_test.go create mode 100644 tests/dovecot_sasld_test.go create mode 100644 tests/gocovcat.go create mode 100644 tests/golangci-noisy.yml create mode 100644 tests/imap_test.go create mode 100644 tests/imapsql_test.go create mode 100644 tests/issue327_test.go create mode 100644 tests/limits_test.go create mode 100644 tests/lmtp_test.go create mode 100644 tests/mta_test.go create mode 100644 tests/multiple_domains_test.go create mode 100644 tests/replace_addr_test.go create mode 100755 tests/run.sh create mode 100644 tests/smtp_autobuffer_test.go create mode 100644 tests/smtp_test.go create mode 100644 tests/stress_test.go create mode 100644 tests/t.go create mode 100755 tests/testdata/check_command.sh create mode 100644 tests/testdata/testing+addHeader@maddy.test.hdr create mode 100644 tests/testdata/testing+reject@maddy.test.exit diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ef99e5a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +testdata/ +cmd/maddy/maddy +maddy +tests/maddy.cover diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..3ee4163 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{scd,go}] +indent_style = tab +indent_size = 4 + +[*.yml] +indent_style = tab +indent_size = 2 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..3108941 --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,52 @@ +# Code of Merit + +**1.** The project creators, lead developers, core team, constitute the managing +members of the project and have final say in every decision of the project, +technical or otherwise, including overruling previous decisions. There are no +limitations to this decisional power. + +**2.** Contributions are an expected result of your membership on the project. +Don’t expect others to do your work or help you with your work forever. + +**3.** All members have the same opportunities to seek any challenge they want +within the project. + +**4.** Authority or position in the project will be proportional to the accrued +contribution. Seniority must be earned. + +**5.** Software is evolutive: the better implementations must supersede lesser +implementations. Technical advantage is the primary evaluation metric. + +**6.** This is a space for technical prowess; topics outside of the project will +not be tolerated. + +**7.** Non technical conflicts will be discussed in a separate space. Disruption +of the project will not be allowed. + +**8.** Individual characteristics, including but not limited to, body, sex, +sexual preference, race, language, religion, nationality, or political +preferences are irrelevant in the scope of the project and will not be taken +into account concerning your value or that of your contribution to the project. + +**9.** Discuss or debate the idea, not the person. + +**10.** There is no room for ambiguity: Ambiguity will be met with questioning; +further ambiguity will be met with silence. It is the responsibility of the +originator to provide requested context. + +**11.** If something is illegal outside the scope of the project, it is illegal +in the scope of the project. This Code of Merit does not take precedence over +governing law. + +**12.** This Code of Merit governs the technical procedures of the project not +the activities outside of it. + +**13.** Participation on the project equates to agreement of this Code of Merit. + +**14.** No objectives beyond the stated objectives of this project are relevant +to the project. Any intent to deviate the project from its original purpose of +existence will constitute grounds for remedial action which may include +expulsion from the project. + +This document is the Code of Merit +(`http://code-of-merit.org`), version 1.0. diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..45bafd9 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,59 @@ +# Contributing Guidelines + +Of course, we love our contributors. Thanks for spending time on making maddy +better. + +## Reporting bugs + +**Issue tracker is meant to be used only if you have a problem or a feature +request. If you just have some questions about maddy - prefer to use the +[IRC channel](https://webchat.oftc.net/?channels=maddy&uio=MT11bmRlZmluZWQb1).** + +- Provide log files, preferably with 'debug' directive set. +- Provide the exact steps to reproduce the issue. +- Provide the example message that causes the error, if applicable. +- "Too much information is better than not enough information". + +Issues without enough information will be ignored and possibly closed. +Take some time to be more useful. + +See SECURITY.md for information on how to report vulnerabilities. + +## Contributing Code + +0. Use common sense. +1. Learn Git. Especially, what is `git rebase`. We may ask you to use it if + needed. +2. Tell us that you are willing to work on an issue. +3. Fork the repo. Create a new branch based on `dev`, write your code. Open a + PR. + +Ask for advice if you are not sure. We don't bite. + +maddy design summary and some recommendations are provided in +[HACKING.md](../HACKING.md) file. + +## Commits + +1. Prefix commit message with a package path if it affects only a single + package. Omit `internal/` for brevity. +2. Provide reasoning for details in the source code itself (via comments), + provide reasoning for high-level decisions in the commit message. +3. Make sure every commit builds & passes tests. Otherwise `git bisect` becomes + unusable. + +## Git workflow + +`dev` branch includes the in-development version for the next feature release. +It is based on commit of the latest stable release and is merged into `master` +on release via fast-forward. Unlike `master`, `dev` **is not a protected branch +and may get force-pushes**. + +`master` branch contains the latest stable release and is frozen between +releases. + +`fix-X.Y` are temporary branches containing backported security fixes. +They are based on the commit of the corresponding stable release and exist +while the corresponding release is maintained. A `fix-*` branch is not created +for the latest release. Changes are added to these branches by cherry-picking +needed commits from the `dev` branch. diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 0000000..310954d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,27 @@ +--- +name: Bug report +about: If you think something is broken +title: Bug report +labels: bug +assignees: '' + +--- + +# Describe the bug + +What do you think is wrong? + +# Steps to reproduce + +# Log files + +Use a service like hastebin.com or attach a file if it is big + +# Configuration file + +Located in /etc/maddy/maddy.conf by default, don't forget to remove DB passwords +and other security-related stuff. + +# Environment information + +* maddy version: ? diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..7573e18 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,7 @@ +contact_links: + - name: Questions + url: "https://github.com/foxcpp/maddy/discussions/new?category=q-a" + about: "Use GitHub discussions for any questions" + - name: IRC channel + url: "https://webchat.oftc.net/?channels=maddy&uio=MT11bmRlZmluZWQb1" + about: "... or there is also an IRC channel for any discussions" diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 0000000..63e29a9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,22 @@ +--- +name: Feature request +about: If you would like to see a new feature in maddy. +title: Feature request +labels: new feature +assignees: '' + +--- + +# Use case + +What problem you are trying to solve? + +Note alternatives you considered and why they are not useful. + +# Your idea for a solution + +How your solution would work in general? +Note that some overly complicated solutions may be rejected because maddy is +meant to be simple. + +- [ ] I'm willing to help with the implementation diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 0000000..150da57 --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,17 @@ +# Security Policy + +## Supported Versions + +Two latest incompatible releases (e.g. 2.0.0 and 1.9.0). + +Latest release gets all bug fixes, features, etc. Previous incompatible release +gets security fixes and fixes for problems that render software completely +unusable in certain configurations with no workaround. + +## Reporting a Vulnerability + +If you believe the vulnerabilitiy does have a big impact on existing +deployments - email `fox.cpp at disroot.org`, put "[maddy security]" in the +Subject. + +Otherwise, open a public issue. diff --git a/.github/releases.md b/.github/releases.md new file mode 100644 index 0000000..b52fa81 --- /dev/null +++ b/.github/releases.md @@ -0,0 +1,41 @@ +# Release preparation + +1. Run linters, fix all simple warnings. If the behavior is intentional - add +`nolint` comment and explanation. If the warning is non-trviail to fix - open +an issue. +``` +golangci-lint run +``` + +2. Run unit tests suite. Verify that all disabled tests are not related to + serious problems and have corresponding issue open. +``` +go test ./... +``` + +3. Run integration tests suite. Verify that all disabled tests are not related + to serious problems and have corresponding issue open. +``` +cd tests/ +./run.sh +``` + +4. Write release notes. + +5. Create PGP-signed Git tag and push it to GitHub (do not create a "release" + yet). + +5. Use environment configuration from maddy-repro bundle + (https://foxcpp.dev/maddy-repro) to build release artifacts. + +6. Create detached PGP signatures for artifacts using key + 3197BBD95137E682A59717B434BB2007081396F4. + +7. Create sha256sums file for artifacts. + +8. Create release on GitHub using the same text for + release notes. Attach signed artifacts and sha256sums file. + +9. Build the Docker container and push it to hub.docker.com. + +10. Post a message on the sr.ht mailing list. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..e7155b6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,158 @@ +name: "Prepare release artifacts" + +on: + push: + tags: [ "v*" ] + +permissions: + id-token: write + contents: read + attestations: write + packages: write + +jobs: + artifact-builder-x86: + name: "Prepare release artifacts (x86)" + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + container: + image: "alpine:edge" + steps: + - uses: actions/checkout@v1 # v2 does not work with containers + - name: "Install build dependencies" + run: | + apk add --no-cache gcc go zstd + - name: "Create and package build tree" + run: | + ./build.sh --builddir ~/package-output/ --static build + ver=$(cat .version) + if [ "v$ver" != "${{github.ref_name}}" ]; then echo ".version does not match the Git tag"; exit 1; fi + mv ~/package-output/ ~/maddy-$ver-x86_64-linux-musl + cd ~ + tar c ./maddy-$ver-x86_64-linux-musl | zstd > ~/maddy-x86_64-linux-musl.tar.zst + cd - + - name: "Save source tree" + run: | + rm -rf .git + ver=$(cat .version) + cp -r . ~/maddy-$ver-src + cd ~ + tar c ./maddy-$ver-src | zstd > ~/maddy-src.tar.zst + cd - + - name: "Upload source tree" + uses: actions/upload-artifact@v4 + with: + name: maddy-src.tar.zst + path: '~/maddy-src.tar.zst' + if-no-files-found: error + - name: "Upload binary tree" + uses: actions/upload-artifact@v4 + with: + name: maddy-binary.tar.zst + path: '~/maddy-x86_64-linux-musl.tar.zst' + if-no-files-found: error + - name: "Generate artifact attestation" + uses: actions/attest-build-provenance@v2 + with: + subject-path: '~/maddy-x86_64-linux-musl.tar.zst' + artifact-builder-arm: + name: "Prepare release artifacts (aarch64)" + if: github.ref_type == 'tag' + runs-on: ubuntu-22.04-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + # Building in a Docker container is a workaround for the issue of + # JavaScript-based GitHub Actions not being supported in Alpine + # containers on the Arm64 platform. Otherwise, we could completely reuse + # artifact-builder-x86 as a matrix job by running it on an Arm runner. + - name: Build in Docker container + run: | + # Create Dockerfile for the build + cat > Dockerfile << 'EOF' + FROM alpine:edge + RUN apk add --no-cache gcc go zstd musl-dev scdoc + WORKDIR /build + COPY . . + RUN ./build.sh --builddir /package-output/ --static build && \ + ver=$(cat .version) && \ + if [ "v$ver" != "${{github.ref_name}}" ]; then echo ".version does not match the Git tag"; exit 1; fi && \ + mv /package-output/ /maddy-$ver-aarch64-linux-musl && \ + cd / && \ + tar c ./maddy-$ver-aarch64-linux-musl | zstd > /maddy-aarch64-linux-musl.tar.zst + EOF + # Build the image, create a temporary container and copy the artifact. + docker build -t maddy-builder . + container_id=$(docker create maddy-builder) + docker cp $container_id:/maddy-aarch64-linux-musl.tar.zst . + docker rm $container_id + - name: Upload binary tree + uses: actions/upload-artifact@v4 + with: + name: maddy-binary-aarch64.tar.zst + path: maddy-aarch64-linux-musl.tar.zst + if-no-files-found: error + - name: "Generate artifact attestation" + uses: actions/attest-build-provenance@v2 + with: + subject-path: 'maddy-aarch64-linux-musl.tar.zst' + docker-builder: + name: "Build & push Docker image" + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: "Set up QEMU" + uses: docker/setup-qemu-action@v1 + with: + platforms: arm64 + - name: "Set up Docker Buildx" + id: buildx + uses: docker/setup-buildx-action@v3 + - name: "Login to Docker Hub" + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + logout: false + - name: "Login to GitHub Container Registry" + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + logout: false # https://news.ycombinator.com/item?id=28607735 + - name: "Generate container metadata" + uses: docker/metadata-action@v5 + id: meta + with: + images: | + foxcpp/maddy + ghcr.io/foxcpp/maddy + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + labels: | + org.opencontainers.image.title=Maddy Mail Server + org.opencontainers.image.documentation=https://maddy.email/docker/ + org.opencontainers.image.url=https://maddy.email + - name: "Build and push" + uses: docker/build-push-action@v6 + id: docker + with: + context: . + platforms: linux/amd64 #,linux/arm64 Temporary disabled due to SIGSEGV in gcc. + file: Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + - name: "Generate container attestation" + uses: actions/attest-build-provenance@v2 + with: + subject-name: ghcr.io/foxcpp/maddy + subject-digest: ${{ steps.docker.outputs.digest }} + push-to-registry: true + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..32c02e5 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,75 @@ +name: "Testing" + +on: + push: + branches: [ master, dev ] + tags: [ "v*" ] + pull_request: + branches: [ master, dev ] + +permissions: + contents: read + pull-requests: read + checks: write + +jobs: + golangci: + name: Lint + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: "Install libpam" + run: | + sudo apt-get update + sudo apt-get install -y libpam-dev + - uses: golangci/golangci-lint-action@v6 + with: + version: v1.60 + args: "--timeout=30m" + buildsh: + name: "Verify build.sh" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: "Install libpam" + run: | + sudo apt-get update + sudo apt-get install -y libpam-dev + - name: "Verify build.sh" + run: | + ./build.sh + ./build.sh --destdir destdir/ install + find destdir/ + test: + name: "Build and test" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: "Install libpam" + run: | + sudo apt-get update + sudo apt-get install -y libpam-dev + - name: "Unit & module tests" + run: | + go test ./... -coverprofile=coverage.out -covermode=atomic + - name: "Integration tests" + run: | + cd tests/ + ./run.sh + - uses: codecov/codecov-action@v2 + with: + files: ./coverage.out + flags: unit + - uses: codecov/codecov-action@v2 + with: + files: ./tests/coverage.out + flags: integration diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..790b848 --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# gitignore.io +*.o +*.a +*.so +_obj +_test +*.[568vq] +[568vq].out +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* +_testmain.go +*.exe +*.exe~ +*.test +*.prof +**/.envrc +**/.DS_Store + +# Tests coverage +*.out + +# Compiled binaries +cmd/maddy/maddy +cmd/maddy-*-helper/maddy-*-helper +/maddy + +# Man pages +docs/man/*.1 +docs/man/*.5 + +# Certificates and private keys. +*.pem +*.crt +*.key + +# Some directories that may be created during test-runs +# in repo directory. +cmd/maddy/*mtasts-cache +cmd/maddy/*queue + +build/ + +tests/maddy.cover +tests/maddy + +.idea/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..8617544 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,17 @@ +linters: + enable: + - gosimple + - errcheck + - staticcheck + - ineffassign + - typecheck + - govet + - unused + - goimports + - prealloc + - unconvert + - misspell + - whitespace + - nakedret + - dogsled + - copyloopvar diff --git a/.mkdocs.yml b/.mkdocs.yml new file mode 100644 index 0000000..3e90eec --- /dev/null +++ b/.mkdocs.yml @@ -0,0 +1,84 @@ +site_name: maddy + +repo_url: https://github.com/foxcpp/maddy + +theme: alb + +markdown_extensions: + - codehilite: + guess_lang: false + +nav: + - faq.md + - Tutorials: + - tutorials/setting-up.md + - tutorials/building-from-source.md + - tutorials/alias-to-remote.md + - tutorials/pam.md + - Release builds: 'https://maddy.email/builds/' + - multiple-domains.md + - upgrading.md + - seclevels.md + - docker.md + - Reference manual: + - reference/modules.md + - reference/global-config.md + - reference/tls.md + - reference/tls-acme.md + - Endpoints configuration: + - reference/endpoints/imap.md + - reference/endpoints/smtp.md + - reference/endpoints/openmetrics.md + - IMAP storage: + - reference/storage/imap-filters.md + - reference/storage/imapsql.md + - Blob storage: + - reference/blob/fs.md + - reference/blob/s3.md + - reference/smtp-pipeline.md + - SMTP targets: + - reference/targets/queue.md + - reference/targets/remote.md + - reference/targets/smtp.md + - SMTP checks: + - reference/checks/actions.md + - reference/checks/dkim.md + - reference/checks/spf.md + - reference/checks/milter.md + - reference/checks/rspamd.md + - reference/checks/dnsbl.md + - reference/checks/command.md + - reference/checks/authorize_sender.md + - reference/checks/misc.md + - SMTP modifiers: + - reference/modifiers/dkim.md + - reference/modifiers/envelope.md + - Lookup tables (string translation): + - reference/table/static.md + - reference/table/regexp.md + - reference/table/file.md + - reference/table/sql_query.md + - reference/table/chain.md + - reference/table/email_localpart.md + - reference/table/email_with_domain.md + - reference/table/auth.md + - Authentication providers: + - reference/auth/pass_table.md + - reference/auth/pam.md + - reference/auth/shadow.md + - reference/auth/external.md + - reference/auth/ldap.md + - reference/auth/dovecot_sasl.md + - reference/auth/plain_separate.md + - reference/auth/netauth.md + - reference/config-syntax.md + - Integration with software: + - third-party/dovecot.md + - third-party/smtp-servers.md + - third-party/rspamd.md + - third-party/mailman3.md + - Internals: + - internals/specifications.md + - internals/unicode.md + - internals/quirks.md + - internals/sqlite.md diff --git a/.version b/.version new file mode 100644 index 0000000..6f4eebd --- /dev/null +++ b/.version @@ -0,0 +1 @@ +0.8.1 diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2da6211 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM golang:1.23-alpine AS build-env + +ARG ADDITIONAL_BUILD_TAGS="" + +RUN set -ex && \ + apk upgrade --no-cache --available && \ + apk add --no-cache build-base + +WORKDIR /maddy + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . ./ +RUN mkdir -p /pkg/data && \ + cp maddy.conf.docker /pkg/data/maddy.conf && \ + ./build.sh --builddir /tmp --destdir /pkg/ --tags "docker ${ADDITIONAL_BUILD_TAGS}" build install + +FROM alpine:3.21.2 +LABEL maintainer="fox.cpp@disroot.org" +LABEL org.opencontainers.image.source=https://github.com/foxcpp/maddy + +RUN set -ex && \ + apk upgrade --no-cache --available && \ + apk --no-cache add ca-certificates +COPY --from=build-env /pkg/data/maddy.conf /data/maddy.conf +COPY --from=build-env /pkg/usr/local/bin/maddy /bin/ + +EXPOSE 25 143 993 587 465 +VOLUME ["/data"] +ENTRYPOINT ["/bin/maddy", "-config", "/data/maddy.conf"] +CMD ["run"] diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 0000000..1394edb --- /dev/null +++ b/HACKING.md @@ -0,0 +1,130 @@ +## Design goals + +- **Make it easy to deploy.** + Minimal configuration changes should be required to get a typical mail server + running. Though, it is important to avoid making guesses for a + "zero-configuration". A wrong guess is worse than no guess. + +- **Provide 80% of needed components.** + E-mail has evolved into a huge mess. With a single package to do one thing, it + quickly turns into a maintenance nightmare. Put all stuff mail server + typically needs into a single package. Though, leave controversial or highly + opinionated stuff out, don't force people to do things our way + (see next point). + +- **Interoperate with existing software.** + Implement (de-facto) standard protocols not only for clients but also for + various server-side helper software (content filters, etc). + +- **Be secure but interoperable.** + Verify DKIM signatures by default, use DMRAC policies by default, etc. This + makes default setup as secure as possible while maintaining reasonable + interoperability. Though, users can configure maddy to be stricter. + +- **Achieve flexibility through composability.** + Allow connecting components in arbitrary ways instead of restricting users to + predefined templates. + +- **Use Go concurrency features to the full extent.** + Do as much I/O as possible in parallel to minimize latencies. It is silly to + not do so when it is possible. + +## Design summary + +Here is a summary of how things are organized in maddy in general. It explains +things from the developer perspective and is meant to be used as an +introduction by the new developers/contributors. It is recommended to read +user documentation to understand how things work from the user perspective as +well. + +- User documentation: [maddy.conf(5)](docs/man/maddy.5.scd) +- Design rationale: [Comments on design (Wiki)][1] + +There are components called "modules". They are represented by objects +implementing the module.Module interface. Each module gets its unique name. +The function used to create a module instance is saved with this name as a key +into the global map called "modules registry". + +Whenever module needs another module for some functionality, it references it +using a configuration directive with a matcher that internally calls +`modconfig.ModuleFromNode`. That function looks up the module "constructor" in +the registry, calls it with corresponding arguments, checks whether the +returned module satisfies the needed interfaces and then initializes it. + +Alternatively, if configuration uses &-syntax to reference existing +configuration block, `ModuleFromNode` simply looks it up in the global instances +registry. All modules defined the configuration as a separate top-level blocks +are created before main initialization and are placed in the instances registry +where they can be looked up as mentioned before. + +Top-level defined module instances are initialized (`Init` method) lazily as +they are required by other modules. 'smtp' and 'imap' modules follow a special +initialization path, so they are always initialized directly. + +## Error handling + +Familiarize yourself with the `github.com/foxcpp/maddy/framework/exterrors` +package and make sure you have the following for returned errors: +- SMTP status information (smtp\_code, smtp\_enchcode, smtp\_msg fields) + - SMTP message text should contain a generic description of the error + condition without any details to prevent accidental disclosure of the + server configuration details. +- `Temporary() == true` for temporary errors (see `exterrors.WithTemporary`) +- Field that includes the module name + +The easiest way to get all of these is to use `exterrors.SMTPError`. +Put the original error into the `Err` field, so it can be inspected using +`errors.Is`, `errors.Unwrap`, etc. Put the module name into `CheckName` or +`TargetName`. Add any additional context information using the `Misc` field. +Note, the SMTP status code overrides the result of `exterrors.IsTemporary()` +for that error object, so set it using `exterrors.SMTPCode` that uses +`IsTemporary` to select between two codes. + +If the error you are wrapping contains details in its structure fields (like +`*net.OpError`) - copy these values into `Misc` map, put the underlying error +object (`net.OpError.Err`, for example) into the `Err` field. +Avoid using `Reason` unless you are sure you can provide the error message +better than the `Err.Error()` or `Err` is `nil`. + +Do not attempt to add a SMTP status information for every single possible +error. Use `exterrors.WithFields` with basic information for errors you don't +expect. The SMTP client will get the "Internal server error" message and this +is generally the right thing to do on unexpected errors. + +### Goroutines and panics + +If you start any goroutines - make sure to catch panics to make sure severe +bugs will not bring the whole server down. + +## Adding a check + +"Check" is a module that inspects the message and flags it as spam or rejects +it altogether based on some condition. + +The skeleton for the stateful check module can be found in +`internal/check/skeleton.go`. Throw it into a file in +`internal/check/check_name` directory and start ~~breaking~~ extending it. + +If you don't need any per-message state, you can use `StatelessCheck` wrapper. +See `check/dns` directory for a working example. + +Here are some guidelines to make sure your check works well: +- RTFM, docs will tell you about any caveats. +- Don't share any state _between_ messages, your code will be executed in + parallel. +- Use `github.com/foxcpp/maddy/check.FailAction` to select behavior on check + failures. See other checks for examples on how to use it. +- You can assume that order of check functions execution is as follows: + `CheckConnection`, `CheckSender`, `CheckRcpt`, `CheckBody`. + +## Adding a modifier + +"Modifier" is a module that can modify some parts of the message data. + +Note, currently this is not possible to modify the body contents, only header +can be modified. + +Structure of the modifier implementation is similar to the structure of check +implementation, check `modify/replace\_addr.go` for a working example. + +[1]: https://github.com/foxcpp/maddy/wiki/Dev:-Comments-on-design diff --git a/README.md b/README.md new file mode 100644 index 0000000..312fb84 --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +Maddy Mail Server +===================== +> Composable all-in-one mail server. + +Maddy Mail Server implements all functionality required to run a e-mail +server. It can send messages via SMTP (works as MTA), accept messages via SMTP +(works as MX) and store messages while providing access to them via IMAP. +In addition to that it implements auxiliary protocols that are mandatory +to keep email reasonably secure (DKIM, SPF, DMARC, DANE, MTA-STS). + +It replaces Postfix, Dovecot, OpenDKIM, OpenSPF, OpenDMARC and more with one +daemon with uniform configuration and minimal maintenance cost. + +**Note:** IMAP storage is "beta". If you are looking for stable and +feature-packed implementation you may want to use Dovecot instead. maddy still +can handle message delivery business. + +[![CI status](https://img.shields.io/github/actions/workflow/status/foxcpp/maddy/cicd.yml?style=flat-square)](https://github.com/foxcpp/maddy/actions/workflows/cicd.yml) +[![Issues tracker](https://img.shields.io/github/issues/foxcpp/maddy?style=flat-square)](https://github.com/foxcpp/maddy) + +* [Setup tutorial](https://maddy.email/tutorials/setting-up/) +* [Documentation](https://maddy.email/) + +* [IRC channel](https://webchat.oftc.net/?channels=maddy&uio=MT11bmRlZmluZWQb1) +* [Mailing list](https://lists.sr.ht/~foxcpp/maddy) diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..419dc1f --- /dev/null +++ b/build.sh @@ -0,0 +1,201 @@ +#!/bin/sh + +destdir=/ +builddir="$PWD/build" +prefix=/usr/local +version= +static=0 +if [ "${GOFLAGS}" = "" ]; then + GOFLAGS="-trimpath" # set some flags to avoid passing "" to go +fi + +print_help() { + cat >&2 < build tags to use + --version version tag to embed into executables (default: auto-detect) + +Additional flags for "go build" can be provided using GOFLAGS environment variable. + +Options for ./build.sh install: + --prefix installation prefix (default: $prefix) + --destdir system root (default: $destdir) +EOF +} + +while :; do + case "$1" in + -h|--help) + print_help + exit + ;; + --builddir) + shift + builddir="$1" + ;; + --prefix) + shift + prefix="$1" + ;; + --destdir) + shift + destdir="$1" + ;; + --version) + shift + version="$1" + ;; + --static) + static=1 + ;; + --tags) + shift + tags="$1" + ;; + --) + break + shift + ;; + -?*) + echo "Unknown option: ${1}. See --help." >&2 + exit 2 + ;; + *) + break + esac + shift +done + +configdir="${destdir}etc/maddy" + +if [ "$version" = "" ]; then + version=unknown + if [ -e .version ]; then + version="$(cat .version)" + fi + if [ -e .git ] && command -v git 2>/dev/null >/dev/null; then + version="${version}+$(git rev-parse --short HEAD)" + fi +fi + +set -e + +build_man_pages() { + set +e + if ! command -v scdoc >/dev/null 2>/dev/null; then + echo '-- [!] No scdoc utility found. Skipping man pages building.' >&2 + set -e + return + fi + set -e + + echo '-- Building man pages...' >&2 + + mkdir -p "${builddir}/man" + for f in ./docs/man/*.1.scd; do + scdoc < "$f" > "${builddir}/man/$(basename "$f" .scd)" + done +} + +build() { + mkdir -p "${builddir}" + echo "-- Version: ${version}" >&2 + if [ "$(go env CC)" = "" ]; then + echo '-- [!] No C compiler available. maddy will be built without SQLite3 support and default configuration will be unusable.' >&2 + fi + + if [ "$static" -eq 1 ]; then + echo "-- Building main server executable..." >&2 + # This is literally impossible to specify this line of arguments as part of ${GOFLAGS} + # using only POSIX sh features (and even with Bash extensions I can't figure it out). + go build -trimpath -buildmode pie -tags "$tags osusergo netgo static_build" \ + -ldflags "-extldflags '-fno-PIC -static' -X \"github.com/foxcpp/maddy.Version=${version}\"" \ + -o "${builddir}/maddy" ${GOFLAGS} ./cmd/maddy + else + echo "-- Building main server executable..." >&2 + go build -tags "$tags" -trimpath -ldflags="-X \"github.com/foxcpp/maddy.Version=${version}\"" -o "${builddir}/maddy" ${GOFLAGS} ./cmd/maddy + fi + + build_man_pages + + echo "-- Copying misc files..." >&2 + + mkdir -p "${builddir}/systemd" + cp dist/systemd/*.service "${builddir}/systemd/" + cp maddy.conf "${builddir}/maddy.conf" +} + +install() { + echo "-- Installing built files..." >&2 + + command install -m 0755 -d "${destdir}/${prefix}/bin/" + command install -m 0755 "${builddir}/maddy" "${destdir}/${prefix}/bin/" + command ln -sf maddy "${destdir}/${prefix}/bin/maddyctl" + command install -m 0755 -d "${configdir}" + + + # We do not want to overwrite existing configuration. + # If the file exists, then save it with .default suffix and warn user. + if [ ! -e "${configdir}/maddy.conf" ]; then + command install -m 0644 ./maddy.conf "${configdir}/maddy.conf" + else + echo "-- [!] Configuration file ${configdir}/maddy.conf exists, saving to ${configdir}/maddy.conf.default" >&2 + command install -m 0644 ./maddy.conf "${configdir}/maddy.conf.default" + fi + + # Attempt to install systemd units only for Linux. + # Check is done using GOOS instead of uname -s to account for possible + # package cross-compilation. + # Though go command might be unavailable if build.sh is run + # with sudo and go installation is user-specific, so fallback + # to using uname -s in the end. + set +e + if command -v go >/dev/null 2>/dev/null; then + set -e + if [ "$(go env GOOS)" = "linux" ]; then + command install -m 0755 -d "${destdir}/${prefix}/lib/systemd/system/" + command install -m 0644 "${builddir}"/systemd/*.service "${destdir}/${prefix}/lib/systemd/system/" + fi + else + set -e + if [ "$(uname -s)" = "Linux" ]; then + command install -m 0755 -d "${destdir}/${prefix}/lib/systemd/system/" + command install -m 0644 "${builddir}"/systemd/*.service "${destdir}/${prefix}/lib/systemd/system/" + fi + fi + + if [ -e "${builddir}"/man ]; then + command install -m 0755 -d "${destdir}/${prefix}/share/man/man1/" + for f in "${builddir}"/man/*.1; do + command install -m 0644 "$f" "${destdir}/${prefix}/share/man/man1/" + done + fi +} + +# Old build.sh compatibility +install_pkg() { + echo "-- [!] Replace 'install_pkg' with 'install' in build.sh invocation" >&2 + install +} +package() { + echo "-- [!] Replace 'package' with 'build' in build.sh invocation" >&2 + build +} + +if [ $# -eq 0 ]; then + build +else + for arg in "$@"; do + eval "$arg" + done +fi diff --git a/cmd/README.md b/cmd/README.md new file mode 100644 index 0000000..3132fea --- /dev/null +++ b/cmd/README.md @@ -0,0 +1,11 @@ +maddy executables +------------------- + +### maddy + +Main server executable. + +### maddy-pam-helper, maddy-shadow-helper + +Utilities compatible with the auth.external module that call libpam or read +/etc/shadow on Unix systems. diff --git a/cmd/maddy-pam-helper/README.md b/cmd/maddy-pam-helper/README.md new file mode 100644 index 0000000..2bfb179 --- /dev/null +++ b/cmd/maddy-pam-helper/README.md @@ -0,0 +1,65 @@ +## maddy-pam-helper + +External setuid binary for interaction with shadow passwords database or other +privileged objects necessary to run PAM authentication. + +### Building + +It is really easy to build it using any GCC: +``` +gcc pam.c main.c -lpam -o maddy-pam-helper +``` + +Yes, it is not a Go binary. + + +### Installation + +maddy-pam-helper is kinda dangerous binary and should not be allowed to be +executed by everybody but maddy's user. At the same moment it needs to have +access to read-protected files. For this reason installation should be done +very carefully to make sure to not introduce any security "holes". + +#### First method + +```shell +chown maddy: /usr/bin/maddy-pam-helper +chmod u+x,g-x,o-x /usr/bin/maddy-pam-helper +``` + +Also maddy-pam-helper needs access to /etc/shadow, one of the ways to provide +it is to set file capability CAP_DAC_READ_SEARCH: +``` +setcap cap_dac_read_search+ep /usr/bin/maddy-pam-helper +``` + +#### Second method + +Another, less restrictive is to make it setuid-root (assuming you have both maddy user and group): +``` +chown root:maddy /usr/bin/maddy-pam-helper +chmod u+xs,g+x,o-x /usr/bin/maddy-pam-helper +``` + +#### Third method + +The best way actually is to create `shadow` group and grant access to +/etc/shadow to it and then make maddy-pam-helper setgid-shadow: +``` +groupadd shadow +chown :shadow /etc/shadow +chmod g+r /etc/shadow +chown maddy:shadow /usr/bin/maddy-pam-helper +chmod u+x,g+xs /usr/bin/maddy-pam-helper +``` + +Pick what works best for you. + +### PAM service + +maddy-pam-helper uses custom service instead of pretending to be su or sudo. +Because of this you should configure PAM to accept it. + +Minimal example using local passwd/shadow database for authentication can be +found in [maddy.conf][maddy.conf] file. +It should be put into /etc/pam.d/maddy. diff --git a/cmd/maddy-pam-helper/maddy.conf b/cmd/maddy-pam-helper/maddy.conf new file mode 100644 index 0000000..c5bef88 --- /dev/null +++ b/cmd/maddy-pam-helper/maddy.conf @@ -0,0 +1,3 @@ +#%PAM-1.0 +auth required pam_unix.so +account required pam_unix.so diff --git a/cmd/maddy-pam-helper/main.c b/cmd/maddy-pam-helper/main.c new file mode 100644 index 0000000..aec687d --- /dev/null +++ b/cmd/maddy-pam-helper/main.c @@ -0,0 +1,51 @@ +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include "pam.h" + +/* +I really doubt it is a good idea to bring Go to the binary whose primary task +is to call libpam using CGo anyway. +*/ + +int run(void) { + char *username = NULL, *password = NULL; + size_t username_buf_len = 0, password_buf_len = 0; + + ssize_t username_len = getline(&username, &username_buf_len, stdin); + if (username_len < 0) { + perror("getline username"); + return 2; + } + + ssize_t password_len = getline(&password, &password_buf_len, stdin); + if (password_len < 0) { + perror("getline password"); + return 2; + } + + // Cut trailing \n. + if (username_len > 0) { + username[username_len - 1] = 0; + } + if (password_len > 0) { + password[password_len - 1] = 0; + } + + struct error_obj err = run_pam_auth(username, password); + if (err.status != 0) { + if (err.status == 2) { + fprintf(stderr, "%s: %s\n", err.func_name, err.error_msg); + } + return err.status; + } + + return 0; +} + +#ifndef CGO +int main() { + return run(); +} +#endif diff --git a/cmd/maddy-pam-helper/main.go b/cmd/maddy-pam-helper/main.go new file mode 100644 index 0000000..0a3879c --- /dev/null +++ b/cmd/maddy-pam-helper/main.go @@ -0,0 +1,38 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package main + +/* +#cgo LDFLAGS: -lpam +#cgo CFLAGS: -DCGO -Wall -Wextra -Werror -Wno-unused-parameter -Wno-error=unused-parameter -Wpedantic -std=c99 +extern int run(); +*/ +import "C" +import "os" + +/* +Apparently, some people would not want to build it manually by calling GCC. +Here we do it for them. Not going to tell them that resulting file is 800KiB +bigger than one built using only C compiler. +*/ + +func main() { + i := int(C.run()) + os.Exit(i) +} diff --git a/cmd/maddy-pam-helper/pam.c b/cmd/maddy-pam-helper/pam.c new file mode 100644 index 0000000..4829164 --- /dev/null +++ b/cmd/maddy-pam-helper/pam.c @@ -0,0 +1,100 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2022 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include "pam.h" + +static int conv_func(int num_msg, const struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) { + struct pam_response *reply = malloc(sizeof(struct pam_response)); + if (reply == NULL) { + return PAM_CONV_ERR; + } + + char* password_cpy = malloc(strlen((char*)appdata_ptr)+1); + if (password_cpy == NULL) { + return PAM_CONV_ERR; + } + memcpy(password_cpy, (char*)appdata_ptr, strlen((char*)appdata_ptr)+1); + + reply->resp = password_cpy; + reply->resp_retcode = 0; + + // PAM frees pam_response for us. + *resp = reply; + + return PAM_SUCCESS; +} + +struct error_obj run_pam_auth(const char *username, char *password) { + const struct pam_conv local_conv = { conv_func, password }; + pam_handle_t *local_auth = NULL; + int status = pam_start("maddy", username, &local_conv, &local_auth); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + ret_val.status = 2; + ret_val.func_name = "pam_start"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + status = pam_authenticate(local_auth, PAM_SILENT|PAM_DISALLOW_NULL_AUTHTOK); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + if (status == PAM_AUTH_ERR || status == PAM_USER_UNKNOWN) { + ret_val.status = 1; + } else { + ret_val.status = 2; + } + ret_val.func_name = "pam_authenticate"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + status = pam_acct_mgmt(local_auth, PAM_SILENT|PAM_DISALLOW_NULL_AUTHTOK); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + if (status == PAM_AUTH_ERR || status == PAM_USER_UNKNOWN || status == PAM_NEW_AUTHTOK_REQD) { + ret_val.status = 1; + } else { + ret_val.status = 2; + } + ret_val.func_name = "pam_acct_mgmt"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + status = pam_end(local_auth, status); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + ret_val.status = 2; + ret_val.func_name = "pam_end"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + struct error_obj ret_val; + ret_val.status = 0; + ret_val.func_name = NULL; + ret_val.error_msg = NULL; + return ret_val; +} + diff --git a/cmd/maddy-pam-helper/pam.h b/cmd/maddy-pam-helper/pam.h new file mode 100644 index 0000000..e9831ec --- /dev/null +++ b/cmd/maddy-pam-helper/pam.h @@ -0,0 +1,27 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +#pragma once + +struct error_obj { + int status; + const char* func_name; + const char* error_msg; +}; + +struct error_obj run_pam_auth(const char *username, char *password); diff --git a/cmd/maddy-shadow-helper/README.md b/cmd/maddy-shadow-helper/README.md new file mode 100644 index 0000000..2202420 --- /dev/null +++ b/cmd/maddy-shadow-helper/README.md @@ -0,0 +1,47 @@ +## maddy-shadow-helper + +External helper binary for interaction with shadow passwords database. +Unlike maddy-pam-helper it supports only local shadow database but it does +not have any C dependencies. + +### Installation + +maddy-shadow-helper is kinda dangerous binary and should not be allowed to be +executed by everybody but maddy's user. At the same moment it needs to have +access to read-protected files. For this reason installation should be done +very carefully to make sure to not introduce any security "holes". + +#### First method + +```shell +chown maddy: /usr/bin/maddy-shadow-helper +chmod u+x,g-x,o-x /usr/bin/maddy-shadow-helper +``` + +Also maddy-shadow-helper needs access to /etc/shadow, one of the ways to provide +it is to set file capability CAP_DAC_READ_SEARCH: +``` +setcap cap_dac_read_search+ep /usr/bin/maddy-shadow-helper +``` + +#### Second method + +Another, less restrictive is to make it setuid-root (assuming you have both maddy user and group): +``` +chown root:maddy /usr/bin/maddy-shadow-helper +chmod u+xs,g+x,o-x /usr/bin/maddy-shadow-helper +``` + +#### Third method + +The best way actually is to create `shadow` group and grant access to +/etc/shadow to it and then make maddy-shadow-helper setgid-shadow: +``` +groupadd shadow +chown :shadow /etc/shadow +chmod g+r /etc/shadow +chown maddy:shadow /usr/bin/maddy-shadow-helper +chmod u+x,g+xs /usr/bin/maddy-shadow-helper +``` + +Pick what works best for you. diff --git a/cmd/maddy-shadow-helper/main.go b/cmd/maddy-shadow-helper/main.go new file mode 100644 index 0000000..0c6bcd5 --- /dev/null +++ b/cmd/maddy-shadow-helper/main.go @@ -0,0 +1,71 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package main + +import ( + "bufio" + "errors" + "fmt" + "os" + + "github.com/foxcpp/maddy/internal/auth/shadow" +) + +func main() { + scnr := bufio.NewScanner(os.Stdin) + + if !scnr.Scan() { + fmt.Fprintln(os.Stderr, scnr.Err()) + os.Exit(2) + } + username := scnr.Text() + + if !scnr.Scan() { + fmt.Fprintln(os.Stderr, scnr.Err()) + os.Exit(2) + } + password := scnr.Text() + + ent, err := shadow.Lookup(username) + if err != nil { + if errors.Is(err, shadow.ErrNoSuchUser) { + os.Exit(1) + } + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + + if !ent.IsAccountValid() { + fmt.Fprintln(os.Stderr, "account is expired") + os.Exit(1) + } + + if !ent.IsPasswordValid() { + fmt.Fprintln(os.Stderr, "password is expired") + os.Exit(1) + } + + if err := ent.VerifyPassword(password); err != nil { + if errors.Is(err, shadow.ErrWrongPassword) { + os.Exit(1) + } + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/maddy/main.go b/cmd/maddy/main.go new file mode 100644 index 0000000..1007417 --- /dev/null +++ b/cmd/maddy/main.go @@ -0,0 +1,29 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package main + +import ( + _ "github.com/foxcpp/maddy" + maddycli "github.com/foxcpp/maddy/internal/cli" + _ "github.com/foxcpp/maddy/internal/cli/ctl" +) + +func main() { + maddycli.Run() +} diff --git a/config.go b/config.go new file mode 100644 index 0000000..2f57b47 --- /dev/null +++ b/config.go @@ -0,0 +1,120 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package maddy + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" +) + +/* +Config matchers for module interfaces. +*/ + +// logOut structure wraps log.Output and preserves +// configuration directive it was constructed from, allowing +// dynamic reinitialization for purposes of log file rotation. +type logOut struct { + args []string + log.Output +} + +func logOutput(_ *config.Map, node config.Node) (interface{}, error) { + if len(node.Args) == 0 { + return nil, config.NodeErr(node, "expected at least 1 argument") + } + if len(node.Children) != 0 { + return nil, config.NodeErr(node, "can't declare block here") + } + + return LogOutputOption(node.Args) +} + +func LogOutputOption(args []string) (log.Output, error) { + outs := make([]log.Output, 0, len(args)) + for i, arg := range args { + switch arg { + case "stderr": + outs = append(outs, log.WriterOutput(os.Stderr, false)) + case "stderr_ts": + outs = append(outs, log.WriterOutput(os.Stderr, true)) + case "syslog": + syslogOut, err := log.SyslogOutput() + if err != nil { + return nil, fmt.Errorf("failed to connect to syslog daemon: %v", err) + } + outs = append(outs, syslogOut) + case "off": + if len(args) != 1 { + return nil, errors.New("'off' can't be combined with other log targets") + } + return log.NopOutput{}, nil + default: + // Log file paths are converted to absolute to make sure + // we will be able to recreate them in right location + // after changing working directory to the state dir. + absPath, err := filepath.Abs(arg) + if err != nil { + return nil, err + } + // We change the actual argument, so logOut object will + // keep the absolute path for reinitialization. + args[i] = absPath + + w, err := os.OpenFile(absPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o666) + if err != nil { + return nil, fmt.Errorf("failed to create log file: %v", err) + } + + outs = append(outs, log.WriteCloserOutput(w, true)) + } + } + + if len(outs) == 1 { + return logOut{args, outs[0]}, nil + } + return logOut{args, log.MultiOutput(outs...)}, nil +} + +func defaultLogOutput() (interface{}, error) { + return log.DefaultLogger.Out, nil +} + +func reinitLogging() { + out, ok := log.DefaultLogger.Out.(logOut) + if !ok { + log.Println("Can't reinitialize logger because it was replaced before, this is a bug") + return + } + + newOut, err := LogOutputOption(out.args) + if err != nil { + log.Println("Can't reinitialize logger:", err) + return + } + + out.Close() + + log.DefaultLogger.Out = newOut +} diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000..990fced --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,6 @@ +# Community contributed resources + +Disclaimer: Nothing inside subdirectories here is directly supported by Maddy +Mail Server maintainers. Some community members may be able to help you or not. + +- Kubernetes helm chart is maintained by @acim. diff --git a/contrib/kubernetes/chart/.helmignore b/contrib/kubernetes/chart/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/contrib/kubernetes/chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/contrib/kubernetes/chart/Chart.yaml b/contrib/kubernetes/chart/Chart.yaml new file mode 100644 index 0000000..85468f8 --- /dev/null +++ b/contrib/kubernetes/chart/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: maddy +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.2.6 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 0.4.0 diff --git a/contrib/kubernetes/chart/README.md b/contrib/kubernetes/chart/README.md new file mode 100644 index 0000000..14e2e74 --- /dev/null +++ b/contrib/kubernetes/chart/README.md @@ -0,0 +1,69 @@ +# maddy Helm chart for Kubernetes + +![Version: 0.2.5](https://img.shields.io/badge/Version-0.2.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.4.1](https://img.shields.io/badge/AppVersion-0.4.1-informational?style=flat-square) + +This is just initial effort to run maddy within Kubernetes cluster. We have used Deployment resource which has some downsides +but at least this chart will allow you to install maddy relatively easily on your Kubernetes cluster. We have considered +StatefulSet and DaemonSet but such solutions would require much more configuration and in casae of DaemonSet also a TCP +load balancer in front of the nodes. + +## Requirement + +In order to run maddy properly, you need to have TLS secret under name maddy present in the cluster. If you have commercial +certificate, you can create it by the following command: + +```sh +kubectl create secret tls maddy --cert=fullchain.pem --key=privkey.pem +``` + +If you use cert-manager, just create the secret under name maddy. + +## Replication + +Default for this chart is 1 replica of maddy. If you try to increase this, you will probably get an error because of +the busy ports 25, 143, 587, etc. We do not support this feature at the moment, so please use just 1 replica. Like said +at the beginning of this document, multiple replicas would probably require to switch do DaemonSet which would further require +to have TCP load balancer and shared storage between all replicas. This is not supported by this chart, sorry. +This chart is used on one node cluster and then installation is straight forward, like described bellow, but if you have +multiple node cluster, please use taints and tolerations to select the desired node. This chart supports tolerations to +be set. + +## Configuration + +| Key | Type | Default | Description | +| -------------------------- | ------ | ----------------- | ----------- | +| affinity | object | `{}` | | +| fullnameOverride | string | `""` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"foxcpp/maddy"` | | +| image.tag | string | `""` | | +| imagePullSecrets | list | `[]` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| persistence.accessMode | string | `"ReadWriteOnce"` | | +| persistence.annotations | object | `{}` | | +| persistence.enabled | bool | `false` | | +| persistence.path | string | `"/data"` | | +| persistence.size | string | `"128Mi"` | | +| podAnnotations | object | `{}` | | +| podSecurityContext | object | `{}` | | +| replicaCount | int | `1` | | +| resources | object | `{}` | | +| securityContext | object | `{}` | | +| service.type | string | `"NodePort"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `""` | | +| tolerations | list | `[]` | | + +## Installing the chart + +```sh +helm upgrade --install maddy ./chart --set service.externapIPs[0]=1.2.3.4 +``` + +1.2.3.4 is your public IP of the node. + +## maddy configuration + +Feel free to tweak files/maddy.conf and files/aliases according to your needs. diff --git a/contrib/kubernetes/chart/files/aliases b/contrib/kubernetes/chart/files/aliases new file mode 100644 index 0000000..a486390 --- /dev/null +++ b/contrib/kubernetes/chart/files/aliases @@ -0,0 +1 @@ +info@example.org: foxcpp@example.org diff --git a/contrib/kubernetes/chart/files/maddy.conf b/contrib/kubernetes/chart/files/maddy.conf new file mode 100644 index 0000000..6e8b66c --- /dev/null +++ b/contrib/kubernetes/chart/files/maddy.conf @@ -0,0 +1,171 @@ +## maddy 0.3 - default configuration file (2020-05-31) +# Suitable for small-scale deployments. Uses its own format for local users DB, +# should be managed via maddy subcommands. +# +# See tutorials at https://foxcpp.dev/maddy for guidance on typical +# configuration changes. +# +# See manual pages (also available at https://foxcpp.dev/maddy) for reference +# documentation. + +# ---------------------------------------------------------------------------- +# Base variables + +$(hostname) = mx1.example.org +$(primary_domain) = example.org +$(local_domains) = $(primary_domain) + +tls file /etc/maddy/certs/fullchain.pem /etc/maddy/certs/privkey.pem + +# ---------------------------------------------------------------------------- +# Local storage & authentication + +# pass_table provides local hashed passwords storage for authentication of +# users. It can be configured to use any "table" module, in default +# configuration a table in SQLite DB is used. +# Table can be replaced to use e.g. a file for passwords. Or pass_table module +# can be replaced altogether to use some external source of credentials (e.g. +# PAM, /etc/shadow file). +# +# If table module supports it (sql_table does) - credentials can be managed +# using 'maddy creds' command. + +auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } +} + +# imapsql module stores all indexes and metadata necessary for IMAP using a +# relational database. It is used by IMAP endpoint for mailbox access and +# also by SMTP & Submission endpoints for delivery of local messages. +# +# IMAP accounts, mailboxes and all message metadata can be inspected using +# imap-* subcommands of maddy. + +storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db +} + +# ---------------------------------------------------------------------------- +# SMTP endpoints + message routing + +hostname $(hostname) + +msgpipeline local_routing { + dmarc yes + check { + require_matching_ehlo + require_mx_record + dkim + spf + } + + # Insert handling for special-purpose local domains here. + # e.g. + # destination lists.example.org { + # deliver_to lmtp tcp://127.0.0.1:8024 + # } + + destination postmaster $(local_domains) { + modify { + replace_rcpt regexp "(.+)\+(.+)@(.+)" "$1@$3" + replace_rcpt file /data/aliases + } + + deliver_to &local_mailboxes + } + + default_destination { + reject 550 5.1.1 "User doesn't exist" + } +} + +smtp tcp://0.0.0.0:25 { + limits { + # Up to 20 msgs/sec across max. 10 SMTP connections. + all rate 20 1s + all concurrency 10 + } + + source $(local_domains) { + reject 501 5.1.8 "Use Submission for outgoing SMTP" + } + default_source { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + reject 550 5.1.1 "User doesn't exist" + } + } +} + +submission tls://0.0.0.0:465 tcp://0.0.0.0:587 { + limits { + # Up to 50 msgs/sec across any amount of SMTP connections. + all rate 50 1s + } + + auth &local_authdb + + source $(local_domains) { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + modify { + dkim $(primary_domain) $(local_domains) default + } + deliver_to &remote_queue + } + } + default_source { + reject 501 5.1.8 "Non-local sender domain" + } +} + +target.remote outbound_delivery { + limits { + # Up to 20 msgs/sec across max. 10 SMTP connections + # for each recipient domain. + destination rate 20 1s + destination concurrency 10 + } + mx_auth { + dane + mtasts { + cache fs + fs_dir mtasts_cache/ + } + local_policy { + min_tls_level encrypted + min_mx_level none + } + } +} + +target.queue remote_queue { + target &outbound_delivery + + autogenerated_msg_domain $(primary_domain) + bounce { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + reject 550 5.0.0 "Refusing to send DSNs to non-local addresses" + } + } +} + +# ---------------------------------------------------------------------------- +# IMAP endpoints + +imap tls://0.0.0.0:993 tcp://0.0.0.0:143 { + auth &local_authdb + storage &local_mailboxes +} diff --git a/contrib/kubernetes/chart/templates/NOTES.txt b/contrib/kubernetes/chart/templates/NOTES.txt new file mode 100644 index 0000000..e69de29 diff --git a/contrib/kubernetes/chart/templates/_helpers.tpl b/contrib/kubernetes/chart/templates/_helpers.tpl new file mode 100644 index 0000000..99b3e0e --- /dev/null +++ b/contrib/kubernetes/chart/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "maddy.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "maddy.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "maddy.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "maddy.labels" -}} +helm.sh/chart: {{ include "maddy.chart" . }} +{{ include "maddy.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "maddy.selectorLabels" -}} +app.kubernetes.io/name: {{ include "maddy.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "maddy.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "maddy.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/contrib/kubernetes/chart/templates/configmap.yaml b/contrib/kubernetes/chart/templates/configmap.yaml new file mode 100644 index 0000000..5d24a25 --- /dev/null +++ b/contrib/kubernetes/chart/templates/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{include "maddy.fullname" .}} + labels: {{- include "maddy.labels" . | nindent 4}} +data: + maddy.conf: | +{{ tpl (.Files.Get "files/maddy.conf") . | printf "%s" | indent 4 }} + aliases: | +{{ tpl (.Files.Get "files/aliases") . | printf "%s" | indent 4 }} diff --git a/contrib/kubernetes/chart/templates/deployment.yaml b/contrib/kubernetes/chart/templates/deployment.yaml new file mode 100644 index 0000000..f0f1492 --- /dev/null +++ b/contrib/kubernetes/chart/templates/deployment.yaml @@ -0,0 +1,113 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "maddy.fullname" . }} + labels: + {{- include "maddy.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "maddy.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ tpl (.Files.Get "files/maddy.conf") . | printf "%s" | sha256sum }} + checksum/aliases: {{ tpl (.Files.Get "files/aliases") . | printf "%s" | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "maddy.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "maddy.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + - name: init + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: busybox + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - cp /tmp/maddy/* /data/. + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: {{ .Values.persistence.path }} + {{- if .Values.persistence.subPath }} + subPath: {{ .Values.persistence.subPath }} + {{- end }} + - name: config + mountPath: /tmp/maddy + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: smtp + containerPort: 25 + protocol: TCP + - name: imaps + containerPort: 993 + protocol: TCP + - name: starttls + containerPort: 587 + protocol: TCP + # livenessProbe: + # httpGet: + # path: / + # port: http + # readinessProbe: + # httpGet: + # path: / + # port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: {{ .Values.persistence.path }} + {{- if .Values.persistence.subPath }} + subPath: {{ .Values.persistence.subPath }} + {{- end }} + - name: tls + mountPath: /etc/maddy/certs/fullchain.pem + subPath: tls.crt + - name: tls + mountPath: /etc/maddy/certs/privkey.pem + subPath: tls.key + volumes: + - name: data + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ default (include "maddy.fullname" .) .Values.persistence.existingClaim }} + {{- else }} + emptyDir: {} + {{- end }} + - name: config + configMap: + name: {{include "maddy.fullname" .}} + - name: tls + secret: + secretName: {{include "maddy.fullname" .}} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/contrib/kubernetes/chart/templates/pvc.yaml b/contrib/kubernetes/chart/templates/pvc.yaml new file mode 100644 index 0000000..a9ffe7f --- /dev/null +++ b/contrib/kubernetes/chart/templates/pvc.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "maddy.fullname" . }} + annotations: + {{- with .Values.persistence.annotations }} + {{ toYaml . | indent 4 }} + {{- end }} + labels: + {{- include "maddy.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.accessMode }} + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass }} + {{- end }} +{{- end -}} + diff --git a/contrib/kubernetes/chart/templates/service.yaml b/contrib/kubernetes/chart/templates/service.yaml new file mode 100644 index 0000000..7320bd3 --- /dev/null +++ b/contrib/kubernetes/chart/templates/service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "maddy.fullname" . }} + labels: + {{- include "maddy.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: 25 + targetPort: smtp + protocol: TCP + name: smtp + - port: 993 + targetPort: imaps + protocol: TCP + name: imaps + - port: 587 + targetPort: starttls + protocol: TCP + name: starttls + selector: + {{- include "maddy.selectorLabels" . | nindent 4 }} + {{- with .Values.service.externalIPs }} + externalIPs: + {{- toYaml . | nindent 6 }} + {{- end -}} diff --git a/contrib/kubernetes/chart/templates/serviceaccount.yaml b/contrib/kubernetes/chart/templates/serviceaccount.yaml new file mode 100644 index 0000000..ace0978 --- /dev/null +++ b/contrib/kubernetes/chart/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "maddy.serviceAccountName" . }} + labels: + {{- include "maddy.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/contrib/kubernetes/chart/templates/tests/test-connection.yaml b/contrib/kubernetes/chart/templates/tests/test-connection.yaml new file mode 100644 index 0000000..bb163b3 --- /dev/null +++ b/contrib/kubernetes/chart/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "maddy.fullname" . }}-test-connection" + labels: + {{- include "maddy.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "maddy.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/contrib/kubernetes/chart/values.yaml b/contrib/kubernetes/chart/values.yaml new file mode 100644 index 0000000..ede78ca --- /dev/null +++ b/contrib/kubernetes/chart/values.yaml @@ -0,0 +1,74 @@ +# Default values for maddy. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 # Multiple replicas are not supported, please don't change this. + +image: + repository: foxcpp/maddy + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: + {} + # fsGroup: 2000 + +securityContext: + {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +# Set externalPIs to your public IP(s) of the node running maddy. In case of multiple nodes, you need to set tolerations +# and taints in order to run maddy on the exact node. +service: + type: NodePort + # externalIPs: + +resources: + {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +persistence: + enabled: false + # existingClaim: "" + accessMode: ReadWriteOnce + size: 128Mi + # storageClass: "" + path: /data + annotations: {} + # subPath: "" # only mount a subpath of the Volume into the pod + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/directories.go b/directories.go new file mode 100644 index 0000000..b3930db --- /dev/null +++ b/directories.go @@ -0,0 +1,46 @@ +//go:build !docker +// +build !docker + +package maddy + +var ( + // ConfigDirectory specifies platform-specific value + // that should be used as a location of default configuration + // + // It should not be changed and is defined as a variable + // only for purposes of modification using -X linker flag. + ConfigDirectory = "/etc/maddy" + + // DefaultStateDirectory specifies platform-specific + // default for StateDirectory. + // + // Most code should use StateDirectory instead since + // it will contain the effective location of the state + // directory. + // + // It should not be changed and is defined as a variable + // only for purposes of modification using -X linker flag. + DefaultStateDirectory = "/var/lib/maddy" + + // DefaultRuntimeDirectory specifies platform-specific + // default for RuntimeDirectory. + // + // Most code should use RuntimeDirectory instead since + // it will contain the effective location of the state + // directory. + // + // It should not be changed and is defined as a variable + // only for purposes of modification using -X linker flag. + DefaultRuntimeDirectory = "/run/maddy" + + // DefaultLibexecDirectory specifies platform-specific + // default for LibexecDirectory. + // + // Most code should use LibexecDirectory since it will + // contain the effective location of the libexec + // directory. + // + // It should not be changed and is defined as a variable + // only for purposes of modification using -X linker flag. + DefaultLibexecDirectory = "/usr/lib/maddy" +) diff --git a/directories_docker.go b/directories_docker.go new file mode 100644 index 0000000..4869f60 --- /dev/null +++ b/directories_docker.go @@ -0,0 +1,11 @@ +//go:build docker +// +build docker + +package maddy + +var ( + ConfigDirectory = "/data" + DefaultStateDirectory = "/data" + DefaultRuntimeDirectory = "/tmp" + DefaultLibexecDirectory = "/usr/lib/maddy" +) diff --git a/dist/README.md b/dist/README.md new file mode 100644 index 0000000..d057f0e --- /dev/null +++ b/dist/README.md @@ -0,0 +1,41 @@ +Distribution files for maddy +------------------------------ + +**Disclaimer:** Most of the files here are maintained in a "best-effort" way. +That is, they may break or become outdated from time to time. Caveat emptor. + +## integration + scripts + +These directories provide pre-made configuration snippets suitable for +easy integration with external software. + +Usually, this is what you use when you put `import integration/something` in +your config. + +## systemd unit + +`maddy.service` launches using default config path (/etc/maddy/maddy.conf). +`maddy@.service` launches maddy using custom config path. E.g. +`maddy@foo.service` will use /etc/maddy/foo.conf. + +Additionally, unit files apply strict sandboxing, limiting maddy permissions on +the system to a bare minimum. Subset of these options makes it impossible for +privileged authentication helper binaries to gain required permissions, so you +may have to disable it when using system account-based authentication with +maddy running as a unprivileged user. + +## fail2ban configuration + +Configuration files for use with fail2ban. Assume either `backend = systemd` specified +in system-wide configuration or log file written to /var/log/maddy/maddy.log. + +See https://github.com/foxcpp/maddy/wiki/fail2ban-configuration for details. + +## logrotate configuration + +Meant for logs rotation when logging to file is used. + +## vim ftdetect/ftplugin/syntax files + +Minimal supplement to make configuration files more readable and help you see +typos in directive names. diff --git a/dist/apparmor/dev.foxcpp.maddy b/dist/apparmor/dev.foxcpp.maddy new file mode 100644 index 0000000..9486c76 --- /dev/null +++ b/dist/apparmor/dev.foxcpp.maddy @@ -0,0 +1,38 @@ +# AppArmor profile for maddy daemon. +# vim:syntax=apparmor:ts=2:sw=2:et + +#include + +profile dev.foxcpp.maddy /usr{/local,}/bin/maddy { + #include + #include + #include + /etc/ca-certificates/** r, + + /etc/resolv.conf r, + /proc/sys/net/core/somaxconn r, + /sys/kernel/mm/transparent_hugepage/hpage_pmd_size r, + deny ptrace, + capability net_bind_service, + network tcp, + network unix, + + # systemd process management and Type=notify + signal (receive) peer=unconfined, + signal (receive) peer=/usr/bin/systemd, + unix (create, connect, send, setopt) type=dgram addr=@*, + /run/systemd/notify w, + + /etc/maddy/** r, + owner /run/maddy/ rw, + owner /run/maddy/** rwkl, + owner /var/lib/maddy/ rw, + owner /var/lib/maddy/** rwk, + owner /var/lib/maddy/**.db-{wal,shm} rmk, + + /usr{/local,}/lib/maddy/* PUx, + + /usr{/local,}/bin/maddy{,ctl} rmix, + + #include if exists +} diff --git a/dist/fail2ban/filter.d/maddy-auth.conf b/dist/fail2ban/filter.d/maddy-auth.conf new file mode 100644 index 0000000..bbaf31b --- /dev/null +++ b/dist/fail2ban/filter.d/maddy-auth.conf @@ -0,0 +1,6 @@ +[INCLUDES] +before = common.conf + +[Definition] +failregex = authentication failed\t\{\"reason\":\".*\",\"src_ip\"\:\":\d+\"\,\"username\"\:\".*\"\}$ +journalmatch = _SYSTEMD_UNIT=maddy.service + _COMM=maddy diff --git a/dist/fail2ban/filter.d/maddy-dictonary-attack.conf b/dist/fail2ban/filter.d/maddy-dictonary-attack.conf new file mode 100644 index 0000000..1b233fa --- /dev/null +++ b/dist/fail2ban/filter.d/maddy-dictonary-attack.conf @@ -0,0 +1,7 @@ +[INCLUDES] +before = common.conf + +[Definition] +failregex = smtp\: MAIL FROM error repeated a lot\, possible dictonary attack\t\{\"count\"\:\d+,\"msg_id\":\".+\",\"src_ip\"\:\":\d+\"\}$ + smtp\: too many RCPT errors\, possible dictonary attack\t\{\"msg_id\":\".+\","src_ip":":\d+\"\} +journalmatch = _SYSTEMD_UNIT=maddy.service + _COMM=maddy diff --git a/dist/fail2ban/jail.d/maddy-auth.conf b/dist/fail2ban/jail.d/maddy-auth.conf new file mode 100644 index 0000000..1543246 --- /dev/null +++ b/dist/fail2ban/jail.d/maddy-auth.conf @@ -0,0 +1,5 @@ +[maddy-auth] +port = 993,465,25 +filter = maddy-auth +bantime = 96h +backend = systemd diff --git a/dist/fail2ban/jail.d/maddy-dictonary-attack.conf b/dist/fail2ban/jail.d/maddy-dictonary-attack.conf new file mode 100644 index 0000000..ebeb33f --- /dev/null +++ b/dist/fail2ban/jail.d/maddy-dictonary-attack.conf @@ -0,0 +1,7 @@ +[maddy-dictonary-attack] +port = 993,465,25 +filter = maddy-dictonary-attack +bantime = 72h +maxretry = 3 +findtime = 6h +backend = systemd diff --git a/dist/install.sh b/dist/install.sh new file mode 100755 index 0000000..33cb693 --- /dev/null +++ b/dist/install.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +DESTDIR=$DESTDIR +if [ -z "$PREFIX" ]; then + PREFIX=/usr/local +fi +if [ -z "$FAIL2BANDIR" ]; then + FAIL2BANDIR=/etc/fail2ban +fi +if [ -z "$CONFDIR" ]; then + CONFDIR=/etc/maddy +fi + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $script_dir + +install -Dm 0644 -t "$DESTDIR/$PREFIX/share/vim/vimfiles/ftdetect/" vim/ftdetect/maddy-conf.vim +install -Dm 0644 -t "$DESTDIR/$PREFIX/share/vim/vimfiles/ftplugin/" vim/ftplugin/maddy-conf.vim +install -Dm 0644 -t "$DESTDIR/$PREFIX/share/vim/vimfiles/syntax/" vim/syntax/maddy-conf.vim + +install -Dm 0644 -t "$DESTDIR/$FAIL2BANDIR/jail.d/" fail2ban/jail.d/* +install -Dm 0644 -t "$DESTDIR/$FAIL2BANDIR/filter.d/" fail2ban/filter.d/* + +install -Dm 0644 -t "$DESTDIR/$PREFIX/lib/systemd/system/" systemd/maddy.service systemd/maddy@.service diff --git a/dist/logrotate.d/maddy b/dist/logrotate.d/maddy new file mode 100644 index 0000000..2c264e7 --- /dev/null +++ b/dist/logrotate.d/maddy @@ -0,0 +1,7 @@ +/var/log/maddy/maddy.log { + missingok + su maddy maddy + postrotate + /usr/bin/killall -USR1 maddy + endscript +} diff --git a/dist/systemd/maddy.service b/dist/systemd/maddy.service new file mode 100644 index 0000000..ec1ac29 --- /dev/null +++ b/dist/systemd/maddy.service @@ -0,0 +1,82 @@ +[Unit] +Description=maddy mail server +Documentation=man:maddy(1) +Documentation=man:maddy.conf(5) +Documentation=https://maddy.email +After=network-online.target + +[Service] +Type=notify +NotifyAccess=main + +User=maddy +Group=maddy + +# cd to state directory to make sure any relative paths +# in config will be relative to it unless handled specially. +WorkingDirectory=/var/lib/maddy + +ConfigurationDirectory=maddy +RuntimeDirectory=maddy +StateDirectory=maddy +LogsDirectory=maddy +ReadOnlyPaths=/usr/lib/maddy +ReadWritePaths=/var/lib/maddy + +# Strict sandboxing. You have no reason to trust code written by strangers from GitHub. +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +ProtectKernelTunables=true +ProtectHostname=true +ProtectClock=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 + +# Additional sandboxing. You need to disable all of these options +# for privileged helper binaries (for system auth) to work correctly. +NoNewPrivileges=true +PrivateDevices=true +DeviceAllow=/dev/syslog +RestrictSUIDSGID=true +ProtectKernelModules=true +MemoryDenyWriteExecute=true +RestrictNamespaces=true +RestrictRealtime=true +LockPersonality=true + +# Graceful shutdown with a reasonable timeout. +TimeoutStopSec=7s +KillMode=mixed +KillSignal=SIGTERM + +# Required to bind on ports lower than 1024. +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE + +# Force all files created by maddy to be only readable by it +# and maddy group. +UMask=0007 + +# Bump FD limitations. Even idle mail server can have a lot of FDs open (think +# of idle IMAP connections, especially ones abandoned on the other end and +# slowly timing out). +LimitNOFILE=131072 + +# Limit processes count to something reasonable to +# prevent resources exhausting due to big amounts of helper +# processes launched. +LimitNPROC=512 + +# Restart server on any problem. +Restart=on-failure +# ... Unless it is a configuration problem. +RestartPreventExitStatus=2 + +ExecStart=/usr/local/bin/maddy run + +ExecReload=/bin/kill -USR1 $MAINPID +ExecReload=/bin/kill -USR2 $MAINPID + +[Install] +WantedBy=multi-user.target diff --git a/dist/systemd/maddy@.service b/dist/systemd/maddy@.service new file mode 100644 index 0000000..ea60ff8 --- /dev/null +++ b/dist/systemd/maddy@.service @@ -0,0 +1,78 @@ +[Unit] +Description=maddy mail server (using %i.conf) +Documentation=man:maddy(1) +Documentation=man:maddy.conf(5) +Documentation=https://maddy.email +After=network-online.target + +[Service] +Type=notify +NotifyAccess=main + +User=maddy +Group=maddy + +ConfigurationDirectory=maddy +RuntimeDirectory=maddy +StateDirectory=maddy +LogsDirectory=maddy +ReadOnlyPaths=/usr/lib/maddy +ReadWritePaths=/var/lib/maddy + +# Strict sandboxing. You have no reason to trust code written by strangers from GitHub. +PrivateTmp=true +PrivateHome=true +ProtectSystem=strict +ProtectKernelTunables=true +ProtectHostname=true +ProtectClock=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +DeviceAllow=/dev/syslog + +# Additional sandboxing. You need to disable all of these options +# for privileged helper binaries (for system auth) to work correctly. +NoNewPrivileges=true +PrivateDevices=true +RestrictSUIDSGID=true +ProtectKernelModules=true +MemoryDenyWriteExecute=true +RestrictNamespaces=true +RestrictRealtime=true +LockPersonality=true + +# Graceful shutdown with a reasonable timeout. +TimeoutStopSec=7s +KillMode=mixed +KillSignal=SIGTERM + +# Required to bind on ports lower than 1024. +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE + +# Force all files created by maddy to be only readable by it and +# maddy group. +UMask=0007 + +# Bump FD limitations. Even idle mail server can have a lot of FDs open (think +# of idle IMAP connections, especially ones abandoned on the other end and +# slowly timing out). +LimitNOFILE=131072 + +# Limit processes count to something reasonable to +# prevent resources exhausting due to big amounts of helper +# processes launched. +LimitNPROC=512 + +# Restart server on any problem. +Restart=on-failure +# ... Unless it is a configuration problem. +RestartPreventExitStatus=2 + +ExecStart=/usr/local/bin/maddy --config /etc/maddy/%i.conf run + +ExecReload=/bin/kill -USR1 $MAINPID +ExecReload=/bin/kill -USR2 $MAINPID + +[Install] +WantedBy=multi-user.target diff --git a/dist/vim/ftdetect/maddy-conf.vim b/dist/vim/ftdetect/maddy-conf.vim new file mode 100644 index 0000000..672b137 --- /dev/null +++ b/dist/vim/ftdetect/maddy-conf.vim @@ -0,0 +1 @@ +au BufNewFile,BufRead /etc/maddy/*,maddy.conf setf maddy-conf diff --git a/dist/vim/ftplugin/maddy-conf.vim b/dist/vim/ftplugin/maddy-conf.vim new file mode 100644 index 0000000..d5c49c6 --- /dev/null +++ b/dist/vim/ftplugin/maddy-conf.vim @@ -0,0 +1,8 @@ +setlocal commentstring=#\ %s + +" That is convention for maddy configs. Period. +" - fox.cpp (maddy developer) +setlocal expandtab +setlocal tabstop=4 +setlocal softtabstop=4 +setlocal shiftwidth=4 diff --git a/dist/vim/syntax/maddy-conf.vim b/dist/vim/syntax/maddy-conf.vim new file mode 100644 index 0000000..d59e799 --- /dev/null +++ b/dist/vim/syntax/maddy-conf.vim @@ -0,0 +1,225 @@ +" vim: noexpandtab ts=4 sw=4 + +if exists("b:current_syntax") + finish +endif + +" Lexer-defined rules +syn match maddyComment "#.*" +syn region maddyString start=+"+ skip=+\\\\\|\\"+ end=+"+ oneline + +syn region maddyBlock start="{" end="}" transparent fold + +hi def link maddyComment Comment +hi def link maddyString String + +" Parser-defined rules +syn match maddyMacroName "[a-z0-9_]" contained containedin=maddyMacro +syn match maddyMacro "$(.\{-})" contains=maddyMacroName + +syn match maddyMacroDefSign "=" contained +syn match maddyMacroDef "\^$([a-z0-9_]\{-})\s=\s.\+" contains=maddyMacro,maddyMacroDefSign + +hi def link maddyMacroName Identifier +hi def link maddyMacro Special +hi def link maddyMacroDefSign Special + +" config.Map values +syn keyword maddyBool yes no + +syn match maddyInt '\<\d\+\>' +syn match maddyInt '\<[-+]\d\+\>' +syn match maddyFloat '\<[-+]\d\+\.\d*\<' + +syn match maddyReference /[ \t]&[^ \t]\+/ms=s+1 contains=maddyReferenceSign +syn match maddyReferenceSign /&/ contained + +hi def link maddyBool Boolean +hi def link maddyInt Number +hi def link maddyFloat Float + +hi def link maddyReferenceSign Special + +" Module values + +" grep --no-file -E 'Register.*\(".+", ' **.go | sed -E 's/.+Register.*\("([^"]+)", .+/\1/' | sort -u +syn keyword maddyModule + \ checks + \ command + \ dane + \ dkim + \ dnsbl + \ dnssec + \ dummy + \ extauth + \ external + \ file + \ identity + \ imap + \ imap_filters + \ imapsql + \ limits + \ lmtp + \ loader + \ local_policy + \ milter + \ modifiers + \ msgpipeline + \ mtasts + \ mx_auth + \ pam + \ pass_table + \ plain_separate + \ queue + \ regexp + \ remote + \ replace_rcpt + \ replace_sender + \ require_matching_rdns + \ require_mx_record + \ require_tls + \ rspamd + \ shadow + \ smtp + \ sql_query + \ sql_table + \ static + \ submission + +syn keyword maddyDispatchDir + \ check + \ modify + \ default_source + \ source + \ default_destination + \ destination + \ reject + \ deliver_to + \ reroute + \ dmarc + +" grep --no-file -E 'cfg..+\(".+", ' **.go | sed -E 's/.+cfg..+\("([^"]+)", .+/\1/' | sort -u +syn keyword maddyModDir + \ add + \ add_header_action + \ allow_multiple_from + \ api_path + \ appendlimit + \ attempt_starttls + \ auth + \ autogenerated_msg_domain + \ body_canon + \ bounce + \ broken_sig_action + \ buffer + \ cache + \ case_insensitive + \ certs + \ check_early + \ client_ipv4 + \ client_ipv6 + \ compression + \ conn_max_idle_count + \ conn_max_idle_time + \ conn_reuse_limit + \ debug + \ defer_sender_reject + \ del + \ domains + \ driver + \ dsn + \ ehlo + \ endpoint + \ enforce_early + \ enforce_testing + \ entry + \ error_resp_action + \ expand_replaceholders + \ fail_action + \ fail_open + \ file + \ flags + \ force_ipv4 + \ fs_dir + \ fsstore + \ full_match + \ hash + \ header_canon + \ helper + \ hostname + \ imap_filter + \ init + \ insecure_auth + \ io_debug + \ io_error_action + \ io_errors + \ junk_mailbox + \ key_column + \ key_path + \ keys + \ limits + \ list + \ local_ip + \ location + \ lookup + \ mailfrom + \ max_logged_rcpt_errors + \ max_message_size + \ max_parallelism + \ max_received + \ max_recipients + \ max_tries + \ min_mx_level + \ min_tls_level + \ mx_auth + \ neutral_action + \ newkey_algo + \ none_action + \ no_sig_action + \ oversign_fields + \ pass + \ perdomain + \ permerr_action + \ quarantine_threshold + \ read_timeout + \ reject_threshold + \ relaxed_requiretls + \ required_fields + \ require_sender_match + \ require_tls + \ requiretls_override + \ responses + \ rewrite_subj_action + \ run_on + \ score + \ selector + \ set + \ settings_id + \ sig_expiry + \ sign_fields + \ sign_subdomains + \ softfail_action + \ SOME_action + \ source + \ sqlite3_busy_timeout + \ sqlite3_cache_size + \ sqlite3_exclusive_lock + \ storage + \ table + \ table_name + \ tag + \ target + \ targets + \ temperr_action + \ tls + \ tls_client + \ use_helper + \ user + \ value_column + \ write_timeout + +hi def link maddyModDir Identifier +hi def link maddyModule Identifier +hi def link maddyDispatchDir Identifier + +let b:current_syntax = "maddy" diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 0000000..d2bc346 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,81 @@ +# Docker + +Official Docker image is available from Docker Hub. + +It expects configuration file to be available at /data/maddy.conf. + +If /data is a Docker volume, then default configuration will be placed there +automatically. If it is used, then MADDY_HOSTNAME, MADDY_DOMAIN environment +variables control the host name and primary domain for the server. TLS +certificate should be placed in /data/tls/fullchain.pem, private key in +/data/tls/privkey.pem + +DKIM keys are generated in /data/dkim_keys directory. + +## Image tags + +- `latest` - A latest stable release. May contain breaking changes. +- `X.Y` - A specific feature branch, it is recommended to use these tags to + receive bugfixes without the risk of feature-related regressions or breaking + changes. +- `X.Y.Z` - A specific stable release + +## Ports + +All standard ports, as described in maddy docs. + +- `25` - SMTP inbound port. +- `465`, `587` - SMTP Submission ports +- `993`, `143` - IMAP4 ports + +## Volumes + +`/data` - maddy state directory. Databases, queues, etc are stored here. You +might want to mount a named volume there. The main configuration file is stored +here too (`/data/maddy.conf`). + +## Management utility + +To run management commands, create a temporary container with the same +/data directory and put the command after the image name, like this: + +``` +docker run --rm -it -v maddydata:/data foxcpp/maddy:0.7 creds create foxcpp@maddy.test +docker run --rm -it -v maddydata:/data foxcpp/maddy:0.7 imap-acct create foxcpp@maddy.test +``` + +Use the same image version as the running server. Things may break badly +otherwise. + +Note that, if you modify messages using maddy subcommands while the server is running - +you must ensure that /tmp from the server is accessible for the management +command. One way to it is to run it using `docker exec` instead of `docker run`: +``` +docker exec -it container_name_here maddy creds create foxcpp@maddy.test +``` + +## Build Tags + +Some Maddy features (such as automatic certificate management via ACME with [a non-default libdns provider](../reference/tls-acme/#dns-providers)) require build tags to be passed to Maddy's `build.sh`, as this is run in the Dockerfile you must compile your own Docker image. Build tags can be set via the docker build argument `ADDITIONAL_BUILD_TAGS` e.g. `docker build --build-arg ADDITIONAL_BUILD_TAGS="libdns_acmedns libdns_route53" -t yourorgname/maddy:yourtagname .`. + + +## TL;DR + +``` +docker volume create maddydata +docker run \ + --name maddy \ + -e MADDY_HOSTNAME=mx.maddy.test \ + -e MADDY_DOMAIN=maddy.test \ + -v maddydata:/data \ + -p 25:25 \ + -p 143:143 \ + -p 465:465 \ + -p 587:587 \ + -p 993:993 \ + foxcpp/maddy:0.7 +``` + +It will fail on first startup. Copy TLS certificate to /data/tls/fullchain.pem +and key to /data/tls/privkey.pem. Run the server again. Finish DNS configuration +(DKIM keys, etc) as described in [tutorials/setting-up/](../tutorials/setting-up/). diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 0000000..b9d755c --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,119 @@ +# Frequently Asked Questions + +## I configured maddy as recommended and gmail still puts my messages in spam + +Unfortunately, GMail policies are opaque so we cannot tell why this happens. + +Verify that you have a rDNS record set for the IP used +by sender server. Also some IPs may just happen to +have bad reputation - check it with various DNSBLs. In this +case you do not have much of a choice but to replace it. + +Additionally, you may try marking multiple messages sent from +your domain as "not spam" in GMail UI. + +## Message sending fails with `dial tcp X.X.X.X:25: connect: connection timed out` in log + +Your provider is blocking outbound SMTP traffic on port 25. + +You either have to ask them to unblock it or forward +all outbound messages via a "smart-host". + +## What is resource usage of maddy? + +For a small personal server, you do not need much more than a +single 1 GiB of RAM and disk space. + +## How to setup a catchall address? + +https://github.com/foxcpp/maddy/issues/243#issuecomment-655694512 + +## maddy command prints a "permission denied" error + +Run maddy command under the same user as maddy itself. +E.g. +``` +sudo -u maddy maddy creds ... +``` + +## How maddy compares to MailCow or Mail-In-The-Box? + +MailCow and MIAB are bundles of well-known email-related software configured to +work together. maddy is a single piece of software implementing subset of what +MailCow and MIAB offer. + +maddy offers more uniform configuration system, more lightweight implementation +and has no dependency on Docker or similar technologies for deployment. + +maddy may have more bugs than 20 years old battle-tested software. + +It is easier to get help with MailCow/MITB since underlying implementations +are well-understood and have active community. + +maddy has no Web interface for administration, that is currently done via CLI +utility. + +## How maddy IMAP server compares to WildDuck? + +Both are "more secure by definition": root access is not required, +implementation is in memory-safe language, etc. + +Both support message compression. + +Both have first-class Unicode/internationalization support. + +WildDuck may offer easier scalability options. maddy does not require you to +setup MongoDB and Redis servers, though. In fact, maddy in its default +configuration has no dependencies besides libc. + +maddy has less builtin authentication providers. This means no +app-specific passwords and all that WildDuck lists under point 4 on their +features page. + +maddy currently has no admin Web interface, all necessary DB changes are +performed via CLI utility. + +## How maddy SMTP server compares to ZoneMTA? + +maddy SMTP server has a lot of similarities to ZoneMTA. +Both have powerful mechanisms for message routing (although designed +differently). + +maddy does not require MongoDB server for deployment. + +maddy has no web interface for queue inspection. However, it can +easily inspected by looking at files in /var/lib/maddy. + +ZoneMTA has a number of features that may make it easier to integrate +with HTTP-based services. maddy speaks standard email protocols (SMTP, +Submission). + +## Is there a webmail? + +No, at least currently. + +I suggest you to check out [alps](https://git.sr.ht/~migadu/alps) if you +are fine with alpha-quality but extremely easy to deploy webmail. + +## Is there a content filter (spam filter)? + +No. maddy moves email messages around, it does not classify +them as bad or good with the notable exception of sender policies. + +It is possible to integrate rspamd using 'rspamd' module. Just add +`rspamd` line to `checks` in `local_routing`, it should just work +in most cases. + +## Is it production-ready? + +maddy is considered "beta" quality. Several people use it for personal email. + +## Single process makes it unreliable. This is dumb! + +This is a compromise between ease of management and reliability. Several +measures are implemented in code base in attempt to reduce possible effect +of bugs in one component. + +Besides, you are not required to use a single process, it is easy to launch +maddy with a non-default configuration path and connect multiple instances +together using off-the-shelf protocols. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..901f723 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,22 @@ +> Composable all-in-one mail server. + +Maddy Mail Server implements all functionality required to run a e-mail +server. It can send messages via SMTP (works as MTA), accept messages via SMTP +(works as MX) and store messages while providing access to them via IMAP. +In addition to that it implements auxiliary protocols that are mandatory +to keep email reasonably secure (DKIM, SPF, DMARC, DANE, MTA-STS). + +It replaces Postfix, Dovecot, OpenDKIM, OpenSPF, OpenDMARC and more with one +daemon with uniform configuration and minimal maintenance cost. + +**Note:** IMAP storage is "beta". If you are looking for stable and +feature-packed implementation you may want to use Dovecot instead. maddy still +can handle message delivery business. + +[![builds.sr.ht status](https://builds.sr.ht/~emersion/maddy.svg)](https://builds.sr.ht/~emersion/maddy?) +[![License text](https://img.shields.io/github/license/foxcpp/maddy)](https://github.com/foxcpp/maddy/blob/master/LICENSE) +[![Issues tracker](https://img.shields.io/github/issues/foxcpp/maddy)](https://github.com/foxcpp/maddy) + +* [Setup tutorial](https://maddy.email/tutorials/setting-up/) +* [IRC channel](https://webchat.oftc.net/?channels=maddy&uio=MT11bmRlZmluZWQb1) +* [Mailing list](https://lists.sr.ht/~foxcpp/maddy) diff --git a/docs/internals/quirks.md b/docs/internals/quirks.md new file mode 100644 index 0000000..bc343cc --- /dev/null +++ b/docs/internals/quirks.md @@ -0,0 +1,23 @@ +# Implementation quirks + +This page documents unusual behavior of the maddy protocols implementations. +Some of these problems break standards, some don't but still can hurt +interoperability. + +## SMTP + +- `for` field is never included in the `Received` header field. + + This is allowed by [RFC 2821]. + +## IMAP + +### `sql` + +- `\Recent` flag is not reset in all cases. + + This _does not_ break [RFC 3501]. Clients relying on it will work (much) less + efficiently. + +[RFC 2821]: https://tools.ietf.org/html/rfc2821 +[RFC 3501]: https://tools.ietf.org/html/rfc3501 diff --git a/docs/internals/specifications.md b/docs/internals/specifications.md new file mode 100644 index 0000000..c042cab --- /dev/null +++ b/docs/internals/specifications.md @@ -0,0 +1,291 @@ +# Followed specifications + +This page lists Internet Standards and other specifications followed by +maddy along with any known deviations. + + +## Message format + +- [RFC 2822] - Internet Message Format +- [RFC 2045] - Multipurpose Internet Mail Extensions (MIME) (part 1) +- [RFC 2046] - Multipurpose Internet Mail Extensions (MIME) (part 2) +- [RFC 2047] - Multipurpose Internet Mail Extensions (MIME) (part 3) +- [RFC 2048] - Multipurpose Internet Mail Extensions (MIME) (part 4) +- [RFC 2049] - Multipurpose Internet Mail Extensions (MIME) (part 5) +- [RFC 6532] - Internationalized Email Headers + +- [RFC 2183] - Communicating Presentation Information in Internet Messages: The + Content-Disposition Header Field + +## IMAP + +- [RFC 3501] - Internet Message Access Protocol - Version 4rev1 + * **Partial**: `\Recent` flag is not reset sometimes. +- [RFC 2152] - UTF-7 + +### Extensions + +- [RFC 2595] - Using TLS with IMAP, POP3 and ACAP +- [RFC 7889] - The IMAP APPENDLIMIT Extension +- [RFC 3348] - The Internet Message Action Protocol (IMAP4). Child Mailbox + Extension +- [RFC 6851] - Internet Message Access Protocol (IMAP) - MOVE Extension +- [RFC 6154] - IMAP LIST Extension for Special-Use Mailboxes + * **Partial**: Only SPECIAL-USE capability. +- [RFC 5255] - Internet Message Access Protocol Internationalization + * **Partial**: Only I18NLEVEL=1 capability. +- [RFC 4978] - The IMAP COMPRESS Extension +- [RFC 3691] - Internet Message Access Protocol (IMAP) UNSELECT command +- [RFC 2177] - IMAP4 IDLE command +- [RFC 7888] - IMAP4 Non-Synchronizing Literals + * LITERAL+ capability. +- [RFC 4959] - IMAP Extension for Simple Authentication and Security Layer + (SASL) Initial Client Response + +## SMTP + +- [RFC 2033] - Local Mail Transfer Protocol +- [RFC 5321] - Simple Mail Transfer Protocol +- [RFC 6409] - Message Submission for Mail + +### Extensions + +- [RFC 1870] - SMTP Service Extension for Message Size Declaration +- [RFC 2920] - SMTP Service Extension for Command Pipelining + * Server support only, not used by SMTP client +- [RFC 2034] - SMTP Service Extension for Returning Enhanced Error Codes +- [RFC 3207] - SMTP Service Extension for Secure SMTP over Transport Layer + Security +- [RFC 4954] - SMTP Service Extension for Authentication +- [RFC 6152] - SMTP Extension for 8-bit MIME +- [RFC 6531] - SMTP Extension for Internationalized Email + +### Misc + +- [RFC 6522] - The Multipart/Report Content Type for the Reporting of Mail + System Administrative Messages +- [RFC 3464] - An Extensible Message Format for Delivery Status Notifications +- [RFC 6533] - Internationalized Delivery Status and Disposition Notifications + +## Email security + +### User authentication + +- [RFC 4422] - Simple Authentication and Security Layer (SASL) +- [RFC 4616] - The PLAIN Simple Authentication and Security Layer (SASL) + Mechanism + +### Sender authentication + +- [RFC 6376] - DomainKeys Identified Mail (DKIM) Signatures +- [RFC 7001] - Message Header Field for Indicating Message Authentication Status +- [RFC 7208] - Sender Policy Framework (SPF) for Authorizing Use of Domains in + Email, Version 1 +- [RFC 7372] - Email Authentication Status Codes +- [RFC 7479] - Domain-based Message Authentication, Reporting, and Conformance + (DMARC) + * **Partial**: No report generation. +- [RFC 8301] - Cryptographic Algorithm and Key Usage Update to DomainKeys + Identified Mail (DKIM) +- [RFC 8463] - A New Cryptographic Signature Method for DomainKeys Identified + Mail (DKIM) +- [RFC 8616] - Email Authentication for Internationalized Mail + +### Recipient authentication + +- [RFC 4033] - DNS Security Introduction and Requirements +- [RFC 6698] - The DNS-Based Authentication of Named Entities (DANE) Transport + Layer Security (TLS) Protocol: TLSA +- [RFC 7672] - SMTP Security via Opportunistic DNS-Based Authentication of + Named Entities (DANE) Transport Layer Security (TLS) +- [RFC 8461] - SMTP MTA Strict Transport Security (MTA-STS) + +## Unicode, encodings, internationalization + +- [RFC 3492] - Punycode: A Bootstring encoding of Unicode for Internationalized + Domain Names in Applications (IDNA) +- [RFC 3629] - UTF-8, a transformation format of ISO 10646 +- [RFC 5890] - Internationalized Domain Names for Applications (IDNA): + Definitions and Document Framework +- [RFC 5891] - Internationalized Domain Names for Applications (IDNA): Protocol +- [RFC 7616] - Preparation, Enforcement, and Comparison of Internationalized + Strings Representing Usernames and Passwords +- [RFC 8264] - PRECIS Framework: Preparation, Enforcement, and Comparison of + Internationalized Strings in Application Protocols +- [Unicode 11.0.0] + - [UAX #15] - Unicode Normalization Forms + +There is a huge list of non-Unicode encodings supported by message parser used +for IMAP static cache and search. See [Unicode support](unicode.md) page for +details. + +## Misc + +- [RFC 5782] - DNS Blacklists and Whitelists + + +[GH 188]: https://github.com/foxcpp/maddy/issues/188 + +[RFC 2822]: https://tools.ietf.org/html/rfc2822 +[RFC 2045]: https://tools.ietf.org/html/rfc2045 +[RFC 2046]: https://tools.ietf.org/html/rfc2046 +[RFC 2047]: https://tools.ietf.org/html/rfc2047 +[RFC 2048]: https://tools.ietf.org/html/rfc2048 +[RFC 2049]: https://tools.ietf.org/html/rfc2049 +[RFC 6532]: https://tools.ietf.org/html/rfc6532 +[RFC 2183]: https://tools.ietf.org/html/rfc2183 +[RFC 3501]: https://tools.ietf.org/html/rfc3501 +[RFC 2152]: https://tools.ietf.org/html/rfc2152 +[RFC 2595]: https://tools.ietf.org/html/rfc2595 +[RFC 7889]: https://tools.ietf.org/html/rfc7889 +[RFC 3348]: https://tools.ietf.org/html/rfc3348 +[RFC 6851]: https://tools.ietf.org/html/rfc6851 +[RFC 6154]: https://tools.ietf.org/html/rfc6154 +[RFC 5255]: https://tools.ietf.org/html/rfc5255 +[RFC 4978]: https://tools.ietf.org/html/rfc4978 +[RFC 3691]: https://tools.ietf.org/html/rfc3691 +[RFC 2177]: https://tools.ietf.org/html/rfc2177 +[RFC 7888]: https://tools.ietf.org/html/rfc7888 +[RFC 4959]: https://tools.ietf.org/html/rfc4959 +[RFC 2033]: https://tools.ietf.org/html/rfc2033 +[RFC 5321]: https://tools.ietf.org/html/rfc5321 +[RFC 6409]: https://tools.ietf.org/html/rfc6409 +[RFC 1870]: https://tools.ietf.org/html/rfc1870 +[RFC 2920]: https://tools.ietf.org/html/rfc2920 +[RFC 2034]: https://tools.ietf.org/html/rfc2034 +[RFC 3207]: https://tools.ietf.org/html/rfc3207 +[RFC 4954]: https://tools.ietf.org/html/rfc4954 +[RFC 6152]: https://tools.ietf.org/html/rfc6152 +[RFC 6531]: https://tools.ietf.org/html/rfc6531 +[RFC 6522]: https://tools.ietf.org/html/rfc6522 +[RFC 3464]: https://tools.ietf.org/html/rfc3464 +[RFC 6533]: https://tools.ietf.org/html/rfc6533 +[RFC 4422]: https://tools.ietf.org/html/rfc4422 +[RFC 4616]: https://tools.ietf.org/html/rfc4616 +[RFC 6376]: https://tools.ietf.org/html/rfc6376 +[RFC 7001]: https://tools.ietf.org/html/rfc7001 +[RFC 7208]: https://tools.ietf.org/html/rfc7208 +[RFC 7372]: https://tools.ietf.org/html/rfc7372 +[RFC 7479]: https://tools.ietf.org/html/rfc7479 +[RFC 8301]: https://tools.ietf.org/html/rfc8301 +[RFC 8463]: https://tools.ietf.org/html/rfc8463 +[RFC 8616]: https://tools.ietf.org/html/rfc8616 +[RFC 4033]: https://tools.ietf.org/html/rfc4033 +[RFC 6698]: https://tools.ietf.org/html/rfc6698 +[RFC 7672]: https://tools.ietf.org/html/rfc7672 +[RFC 8461]: https://tools.ietf.org/html/rfc8461 +[RFC 3492]: https://tools.ietf.org/html/rfc3492 +[RFC 3629]: https://tools.ietf.org/html/rfc3629 +[RFC 5890]: https://tools.ietf.org/html/rfc5890 +[RFC 5891]: https://tools.ietf.org/html/rfc5891 +[RFC 7616]: https://tools.ietf.org/html/rfc7616 +[RFC 8264]: https://tools.ietf.org/html/rfc8264 +[RFC 5782]: https://tools.ietf.org/html/rfc5782 +[RFC 2822]: https://tools.ietf.org/html/rfc2822 +[RFC 2045]: https://tools.ietf.org/html/rfc2045 +[RFC 2046]: https://tools.ietf.org/html/rfc2046 +[RFC 2047]: https://tools.ietf.org/html/rfc2047 +[RFC 2048]: https://tools.ietf.org/html/rfc2048 +[RFC 2049]: https://tools.ietf.org/html/rfc2049 +[RFC 6532]: https://tools.ietf.org/html/rfc6532 +[RFC 3501]: https://tools.ietf.org/html/rfc3501 +[RFC 2595]: https://tools.ietf.org/html/rfc2595 +[RFC 7889]: https://tools.ietf.org/html/rfc7889 +[RFC 3348]: https://tools.ietf.org/html/rfc3348 +[RFC 6851]: https://tools.ietf.org/html/rfc6851 +[RFC 6154]: https://tools.ietf.org/html/rfc6154 +[RFC 5255]: https://tools.ietf.org/html/rfc5255 +[RFC 4978]: https://tools.ietf.org/html/rfc4978 +[RFC 3691]: https://tools.ietf.org/html/rfc3691 +[RFC 2177]: https://tools.ietf.org/html/rfc2177 +[RFC 7888]: https://tools.ietf.org/html/rfc7888 +[RFC 4959]: https://tools.ietf.org/html/rfc4959 +[RFC 2033]: https://tools.ietf.org/html/rfc2033 +[RFC 5321]: https://tools.ietf.org/html/rfc5321 +[RFC 6409]: https://tools.ietf.org/html/rfc6409 +[RFC 1870]: https://tools.ietf.org/html/rfc1870 +[RFC 2920]: https://tools.ietf.org/html/rfc2920 +[RFC 2034]: https://tools.ietf.org/html/rfc2034 +[RFC 3207]: https://tools.ietf.org/html/rfc3207 +[RFC 4954]: https://tools.ietf.org/html/rfc4954 +[RFC 6152]: https://tools.ietf.org/html/rfc6152 +[RFC 6531]: https://tools.ietf.org/html/rfc6531 +[RFC 6522]: https://tools.ietf.org/html/rfc6522 +[RFC 3464]: https://tools.ietf.org/html/rfc3464 +[RFC 6533]: https://tools.ietf.org/html/rfc6533 +[RFC 4422]: https://tools.ietf.org/html/rfc4422 +[RFC 4616]: https://tools.ietf.org/html/rfc4616 +[RFC 6376]: https://tools.ietf.org/html/rfc6376 +[RFC 7001]: https://tools.ietf.org/html/rfc7001 +[RFC 7208]: https://tools.ietf.org/html/rfc7208 +[RFC 7372]: https://tools.ietf.org/html/rfc7372 +[RFC 7479]: https://tools.ietf.org/html/rfc7479 +[RFC 8301]: https://tools.ietf.org/html/rfc8301 +[RFC 8463]: https://tools.ietf.org/html/rfc8463 +[RFC 8616]: https://tools.ietf.org/html/rfc8616 +[RFC 4033]: https://tools.ietf.org/html/rfc4033 +[RFC 6698]: https://tools.ietf.org/html/rfc6698 +[RFC 7672]: https://tools.ietf.org/html/rfc7672 +[RFC 8461]: https://tools.ietf.org/html/rfc8461 +[RFC 3492]: https://tools.ietf.org/html/rfc3492 +[RFC 3629]: https://tools.ietf.org/html/rfc3629 +[RFC 5890]: https://tools.ietf.org/html/rfc5890 +[RFC 5891]: https://tools.ietf.org/html/rfc5891 +[RFC 7616]: https://tools.ietf.org/html/rfc7616 +[RFC 8264]: https://tools.ietf.org/html/rfc8264 +[RFC 5782]: https://tools.ietf.org/html/rfc5782 +[RFC 2822]: https://tools.ietf.org/html/rfc2822 +[RFC 2045]: https://tools.ietf.org/html/rfc2045 +[RFC 2046]: https://tools.ietf.org/html/rfc2046 +[RFC 2047]: https://tools.ietf.org/html/rfc2047 +[RFC 2048]: https://tools.ietf.org/html/rfc2048 +[RFC 2049]: https://tools.ietf.org/html/rfc2049 +[RFC 6532]: https://tools.ietf.org/html/rfc6532 +[RFC 3501]: https://tools.ietf.org/html/rfc3501 +[RFC 2595]: https://tools.ietf.org/html/rfc2595 +[RFC 7889]: https://tools.ietf.org/html/rfc7889 +[RFC 3348]: https://tools.ietf.org/html/rfc3348 +[RFC 6851]: https://tools.ietf.org/html/rfc6851 +[RFC 6154]: https://tools.ietf.org/html/rfc6154 +[RFC 5255]: https://tools.ietf.org/html/rfc5255 +[RFC 4978]: https://tools.ietf.org/html/rfc4978 +[RFC 3691]: https://tools.ietf.org/html/rfc3691 +[RFC 2177]: https://tools.ietf.org/html/rfc2177 +[RFC 7888]: https://tools.ietf.org/html/rfc7888 +[RFC 4959]: https://tools.ietf.org/html/rfc4959 +[RFC 2033]: https://tools.ietf.org/html/rfc2033 +[RFC 5321]: https://tools.ietf.org/html/rfc5321 +[RFC 6409]: https://tools.ietf.org/html/rfc6409 +[RFC 1870]: https://tools.ietf.org/html/rfc1870 +[RFC 2920]: https://tools.ietf.org/html/rfc2920 +[RFC 2034]: https://tools.ietf.org/html/rfc2034 +[RFC 3207]: https://tools.ietf.org/html/rfc3207 +[RFC 4954]: https://tools.ietf.org/html/rfc4954 +[RFC 6152]: https://tools.ietf.org/html/rfc6152 +[RFC 6531]: https://tools.ietf.org/html/rfc6531 +[RFC 6522]: https://tools.ietf.org/html/rfc6522 +[RFC 3464]: https://tools.ietf.org/html/rfc3464 +[RFC 6533]: https://tools.ietf.org/html/rfc6533 +[RFC 4422]: https://tools.ietf.org/html/rfc4422 +[RFC 4616]: https://tools.ietf.org/html/rfc4616 +[RFC 6376]: https://tools.ietf.org/html/rfc6376 +[RFC 8301]: https://tools.ietf.org/html/rfc8301 +[RFC 8463]: https://tools.ietf.org/html/rfc8463 +[RFC 7208]: https://tools.ietf.org/html/rfc7208 +[RFC 7372]: https://tools.ietf.org/html/rfc7372 +[RFC 7479]: https://tools.ietf.org/html/rfc7479 +[RFC 8616]: https://tools.ietf.org/html/rfc8616 +[RFC 4033]: https://tools.ietf.org/html/rfc4033 +[RFC 6698]: https://tools.ietf.org/html/rfc6698 +[RFC 7672]: https://tools.ietf.org/html/rfc7672 +[RFC 8461]: https://tools.ietf.org/html/rfc8461 +[RFC 3492]: https://tools.ietf.org/html/rfc3492 +[RFC 3629]: https://tools.ietf.org/html/rfc3629 +[RFC 5890]: https://tools.ietf.org/html/rfc5890 +[RFC 5891]: https://tools.ietf.org/html/rfc5891 +[RFC 7616]: https://tools.ietf.org/html/rfc7616 +[RFC 8264]: https://tools.ietf.org/html/rfc8264 +[RFC 5782]: https://tools.ietf.org/html/rfc5782 + +[Unicode 11.0.0]: https://www.unicode.org/versions/components-11.0.0.html +[UAX #15]: https://unicode.org/reports/tr15/ diff --git a/docs/internals/sqlite.md b/docs/internals/sqlite.md new file mode 100644 index 0000000..8a397fd --- /dev/null +++ b/docs/internals/sqlite.md @@ -0,0 +1,49 @@ +# maddy & SQLite + +SQLite is a perfect choice for small deployments because no additional +configuration is required to get started. It is recommended for cases when you +have less than 10 mail accounts. + +**Note: SQLite requires DB-wide locking for writing, it means that multiple +messages can't be accepted in parallel. This is not the case for server-based +RDBMS where maddy can accept multiple messages in parallel even for a single +mailbox.** + +## WAL mode + +maddy forces WAL journal mode for SQLite. This makes things reasonably fast and +reduces locking contention which may be important for a typical mail server. + +maddy uses increased WAL autocheckpoint interval. This means that while +maintaining a high write throughput, maddy will have to stop for a bit (0.5-1 +second) every time 78 MiB is written to the DB (with default configuration it +is 15 MiB). + +Note that when moving the database around you need to move WAL journal (`-wal`) +and shared memory (`-shm`) files as well, otherwise some changes to the DB will +be lost. + +## Query planner statistics + +maddy updates query planner statistics on shutdown and every 5 hours. It +provides query planner with information to access the database in more +efficient way because go-imap-sql schema does use a few so called "low-quality +indexes". + +## Auto-vacuum + +maddy turns on SQLite auto-vacuum feature. This means that database file size +will shrink when data is removed (compared to default when it remains unused). + +## Manual vacuuming + +Auto-vacuuming can lead to database fragmentation and thus reduce the read +performance. To do manual vacuum operation to repack and defragment database +file, install the SQLite3 console utility and run the following commands: +``` +sqlite3 -cmd 'vacuum' database_file_path_here.db +sqlite3 -cmd 'pragma wal_checkpoint(truncate)' database_file_path_here.db +``` + +It will take some time to complete, you can close the utility when the +`sqlite>` prompt appears. diff --git a/docs/internals/unicode.md b/docs/internals/unicode.md new file mode 100644 index 0000000..9199762 --- /dev/null +++ b/docs/internals/unicode.md @@ -0,0 +1,96 @@ +# Unicode support + +maddy has the first-class Unicode support in all components (modules). You do +not have to take any actions to make it work with internationalized domains, +mailbox names or non-ASCII message headers. + +Internally, all text fields in maddy are represented in UTF-8 and handled using +Unicode-aware operations for comparisons, case-folding and so on. + +## Non-ASCII data in message headers and bodies + +maddy SMTP implementation does not care about encodings used in MIME headers or +in `Content-Type text/*` charset field. + +However, local IMAP storage implementation needs to perform certain operations +on header contents. This is mostly about SEARCH functionality. For IMAP search +to work correctly, the message body and headers should use one of the following +encodings: + +- ASCII +- UTF-8 +- ISO-8859-1, 2, 3, 4, 9, 10, 13, 14, 15 or 16 +- Windows-1250, 1251 or 1252 (aka Code Page 1250 and so on) +- KOI8-R +- ~~HZGB2312~~, GB18030 +- GBK (aka Code Page 936) +- Shift JIS (aka Code Page 932 or Windows-31J) +- Big-5 (aka Code Page 950) +- EUC-JP +- ISO-2022-JP + +_Support for HZGB2312 is currently disabled due to bugs with security +implications._ + +If mailbox includes a message with any encoding not listed here, it will not +be returned in search results for any request. + +Behavior regarding handling of non-Unicode encodings is not considered stable +and may change between versions (including removal of supported encodings). If +you need your stuff to work correctly - start using UTF-8. + +## Configuration files + +maddy configuration files are assumed to be encoded in UTF-8. Use of any other +encoding will break stuff, do not do it. + +Domain names (e.g. in hostname directive or pipeline rules) can be represented +using the ACE form (aka Punycode). They will be converted to the Unicode form +internally. + +## Local credentials + +'sql' storage backend and authentication provider enforce a number of additional +constraints on used account names. + +PRECIS UsernameCaseMapped profile is enforced for local email addresses. +It limits the use of control and Bidi characters to make sure the used value +can be represented consistently in a variety of contexts. On top of that, the +address is case-folded and normalized to the NFC form for consistent internal +handling. + +PRECIS OpaqueString profile is enforced for passwords. Less strict rules are +applied here. Runs of Unicode whitespace characters are replaced with a single +ASCII space. NFC normalization is applied afterwards. If the resulting string +is empty - the password is not accepted. + +Both profiles are defined in RFC 8265, consult it for details. + +## Protocol support + +### SMTPUTF8 extension + +maddy SMTP implementation includes support for the SMTPUTF8 extension as +defined in RFC 6531. + +This means maddy can handle internationalized mailbox and domain names in MAIL +FROM, RCPT TO commands both for outbound and inbound delivery. + +maddy will not accept messages with non-ASCII envelope addresses unless +SMTPUTF8 support is requested. If a message with SMTPUTF8 flag set is forwarded +to a server without SMTPUTF8 support, delivery will fail unless it is possible +to represent envelope addresses in the ASCII form (only domains use Unicode and +they can be converted to Punycode). Contents of message body (and header) are +not considered and always accepted and sent as-is, no automatic downgrading or +reencoding is done. + +### IMAP UTF8, I18NLEVEL extensions + +Currently, maddy does not include support for UTF8 and I18NLEVEL IMAP +extensions. However, it is not a problem that can prevent it from correctly +handling UTF-8 messages (or even messages in other non-ASCII encodings +mentioned above). + +Clients that want to implement proper handling for Unicode strings may assume +maddy does not handle them properly in e.g. SEARCH commands and so such clients +may download messages and process them locally. diff --git a/docs/man/.gitignore b/docs/man/.gitignore new file mode 100644 index 0000000..ad8dbc8 --- /dev/null +++ b/docs/man/.gitignore @@ -0,0 +1 @@ +_generated_*.md diff --git a/docs/man/README.md b/docs/man/README.md new file mode 100644 index 0000000..2363c6e --- /dev/null +++ b/docs/man/README.md @@ -0,0 +1,16 @@ +maddy manual pages +------------------- + +The reference documentation is maintained in the scdoc format and is compiled +into a set of Unix man pages viewable using the standard `man` utility. + +See https://git.sr.ht/~sircmpwn/scdoc for information about the tool used to +build pages. +It can be used as follows: +``` +scdoc < maddy-filters.5.scd > maddy-filters.5 +man ./maddy-filters.5 +``` + +build.sh script in the repo root compiles and installs man pages if the scdoc +utility is installed in the system. diff --git a/docs/man/maddy.1.scd b/docs/man/maddy.1.scd new file mode 100644 index 0000000..9ae63d5 --- /dev/null +++ b/docs/man/maddy.1.scd @@ -0,0 +1,41 @@ +maddy(1) "maddy mail server" "maddy reference documentation" + +; TITLE Command line arguments + +# Name + +maddy - Composable all-in-one mail server. + +# Synopsis + +*maddy* [options...] + +# Description + +Maddy is Mail Transfer agent (MTA), Mail Delivery Agent (MDA), Mail Submission +Agent (MSA), IMAP server and a set of other essential protocols/schemes +necessary to run secure email server implemented in one executable. + +# Command line arguments + +*-h, -help* + Show help message and exit. + +*-config* _path_ + Path to the configuration file. Default is /etc/maddy/maddy.conf. + +*-libexec* _path_ + Path to the libexec directory. Helper executables will be searched here. + Default is /usr/lib/maddy. + +*-log* _targets..._ + Comma-separated list of logging targets. Valid values are the same as the + 'log' config directive. Affects logging before configuration parsing + completes and after it, if the different value is not specified in the + configuration. + +*-debug* + Enable debug log. You want to use it when reporting bugs. + +*-v* + Print version & build metadata. diff --git a/docs/man/prepare_md.py b/docs/man/prepare_md.py new file mode 100644 index 0000000..9aeb335 --- /dev/null +++ b/docs/man/prepare_md.py @@ -0,0 +1,57 @@ +#!/usr/bin/python3 + +""" +This script does all necessary pre-processing to convert scdoc format into +Markdown. + +Usage: + prepare_md.py < in > out + prepare_md.py file1 file2 file3 + Converts into _generated_file1.md, etc. +""" + +import sys +import re + +anchor_escape = str.maketrans(r' #()./\+-_', '__________') + +def prepare(r, w): + new_lines = list() + title = str() + previous_h1_anchor = '' + + inside_literal = False + + for line in r: + if not inside_literal: + if line.startswith('; TITLE ') and title == '': + title = line[8:] + if line[0] == ';': + continue + # turn *page*(1) into [**page(1)**](../_generated_page.1) + line = re.sub(r'\*(.+?)\*\(([0-9])\)', r'[*\1(\2)*](../_generated_\1.\2)', line) + # *aaa* => **aaa** + line = re.sub(r'\*(.+?)\*', r'**\1**', line) + # remove ++ from line endings + line = re.sub(r'\+\+$', '
', line) + # turn whatever looks like a link into one + line = re.sub(r'(https://[^ \)\(\\]+[a-z0-9_\-])', r'[\1](\1)', line) + # escape underscores inside words + line = re.sub(r'([^ ])_([^ ])', r'\1\\_\2', line) + + if line.startswith('```'): + inside_literal = not inside_literal + + new_lines.append(line) + + if title != '': + print('#', title, file=w) + + print(''.join(new_lines[1:]), file=w) + +if len(sys.argv) == 1: + prepare(sys.stdin, sys.stdout) +else: + for f in sys.argv[1:]: + new_name = '_generated_' + f[:-4] + '.md' + prepare(open(f, 'r'), open(new_name, 'w')) diff --git a/docs/multiple-domains.md b/docs/multiple-domains.md new file mode 100644 index 0000000..46fabf0 --- /dev/null +++ b/docs/multiple-domains.md @@ -0,0 +1,157 @@ +# Multiple domains configuration + +By default, maddy uses email addresses as account identifiers for both +authentication and storage purposes. Therefore, account named `user@example.org` +is completely independent from `user@example.com`. They must be created +separately, may have different credentials and have separate IMAP mailboxes. + +This makes it extremely easy to setup maddy to manage multiple otherwise +independent domains. + +Default configuration file contains two macros - `$(primary_domain)` and +`$(local_domains)`. They are used to used in several places thorough the +file to configure message routing, security checks, etc. + +In general, you should just add all domains you want maddy to manage to +`$(local_domains)`, like this: +``` +$(primary_domain) = example.org +$(local_domains) = $(primary_domain) example.com +``` +Note that you need to pick one domain as a "primary" for use in +auto-generated messages. + +With that done, you can create accounts using both domains in the name, send +and receive messages and so on. Do not forget to configure corresponding SPF, +DMARC and MTA-STS records as was recommended in +the [introduction tutorial](tutorials/setting-up.md). + +Also note that you do not really need a separate TLS certificate for each +managed domain. You can have one hostname e.g. mail.example.org set as an MX +record for multiple domains. + +**If you want multiple domains to share username namespace**, you should change +several more options. + +You can make "user@example.org" and "user@example.com" users share the same +credentials of user "user" but have different IMAP mailboxes ("user@example.org" +and "user@example.com" correspondingly). For that, it is enough to set `auth_map` +globally to use `email_localpart` table: +``` +auth_map email_localpart +``` +This way, when user logs in as "user@example.org", "user" will be passed +to the authentication provider, but "user@example.org" will be passed to the +storage backend. You should create accounts like this: +``` +maddy creds create user +maddy imap-acct create user@example.org +maddy imap-acct create user@example.com +``` + +**If you want accounts to also share the same IMAP storage of account named +"user"**, you can set `storage_map` in IMAP endpoint and `delivery_map` in +storage backend to use `email_locapart`: +``` +storage.imapsql local_mailboxes { + ... + delivery_map email_localpart # deliver "user@*" to "user" +} +imap tls://0.0.0.0:993 { + ... + storage &local_mailboxes + ... + storage_map email_localpart # "user@*" accesses "user" mailbox +} +``` + +You also might want to make it possible to log in without +specifying a domain at all. In this case, use `email_localpart_optional` for +both `auth_map` and `storage_map`. + +You also need to make `authorize_sender` check (used in `submission` endpoint) +accept non-email usernames: +``` +authorize_sender { + ... + user_to_email chain { + step email_localpart_optional # remove domain from username if present + step email_with_domain $(local_domains) # expand username with all allowed domains + } +} +``` + +## TL;DR + +Your options: + +**"user@example.org" and "user@example.com" have distinct credentials and +distinct mailboxes.** + +``` +$(primary_domain) = example.org +$(local_domains) = example.org example.com +``` + +Create accounts as: + +```shell +maddy creds create user@example.org +maddy imap-acct create user@example.org +maddy creds create user@example.com +maddy imap-acct create user@example.com +``` + +**"user@example.org" and "user@example.com" have same credentials but +distinct mailboxes.** + +``` +$(primary_domain) = example.org +$(local_domains) = example.org example.com +auth_map email_localpart +``` + +Create accounts as: +```shell +maddy creds create user +maddy imap-acct create user@example.org +maddy imap-acct create user@example.com +``` + +**"user@example.org", "user@example.com", "user" have same credentials and same +mailboxes.** + +``` + $(primary_domain) = example.org + $(local_domains) = example.org example.com + auth_map email_localpart_optional # authenticating as "user@*" checks credentials for "user" + + storage.imapsql local_mailboxes { + ... + delivery_map email_localpart_optional # deliver "user@*" to "user" mailbox + } + + imap tls://0.0.0.0:993 { + ... + storage_map email_localpart_optional # authenticating as "user@*" accesses "user" mailboxes + } + + submission tls://0.0.0.0:465 { + check { + authorize_sender { + ... + user_to_email chain { + step email_localpart_optional # remove domain from username if present + step email_with_domain $(local_domains) # expand username with all allowed domains + } + } + } + ... + } +``` + +Create accounts as: +```shell +maddy creds create user +maddy imap-acct create user +``` diff --git a/docs/reference/auth/dovecot_sasl.md b/docs/reference/auth/dovecot_sasl.md new file mode 100644 index 0000000..919d42b --- /dev/null +++ b/docs/reference/auth/dovecot_sasl.md @@ -0,0 +1,26 @@ +# Dovecot SASL + +The 'auth.dovecot_sasl' module implements the client side of the Dovecot +authentication protocol, allowing maddy to use it as a credentials source. + +Currently SASL mechanisms support is limited to mechanisms supported by maddy +so you cannot get e.g. SCRAM-MD5 this way. + +``` +auth.dovecot_sasl { + endpoint unix://socket_path +} + +dovecot_sasl unix://socket_path +``` + +## Configuration directives + +### endpoint _schema://address_ +Default: not set + +Set the address to use to contact Dovecot SASL server in the standard endpoint +format. + +`tcp://10.0.0.1:2222` for TCP, `unix:///var/lib/dovecot/auth.sock` for Unix +domain sockets. diff --git a/docs/reference/auth/external.md b/docs/reference/auth/external.md new file mode 100644 index 0000000..9b9659e --- /dev/null +++ b/docs/reference/auth/external.md @@ -0,0 +1,52 @@ +# System command + +auth.external module for authentication using external helper binary. It looks for binary +named `maddy-auth-helper` in $PATH and libexecdir and uses it for authentication +using username/password pair. + +The protocol is very simple: +Program is launched for each authentication. Username and password are written +to stdin, adding \n to the end. If binary exits with 0 status code - +authentication is considered successful. If the status code is 1 - +authentication is failed. If the status code is 2 - another unrelated error has +happened. Additional information should be written to stderr. + +``` +auth.external { + helper /usr/bin/ldap-helper + perdomain no + domains example.org +} +``` + +## Configuration directives + +### helper _file_path_ + +**Required.**
+Location of the helper binary. + +--- + +### perdomain _boolean_ +Default: `no` + +Don't remove domain part of username when authenticating and require it to be +present. Can be used if you want user@domain1 and user@domain2 to be different +accounts. + +--- + +### domains _domains..._ +Default: not specified + +Domains that should be allowed in username during authentication. + +For example, if 'domains' is set to "domain1 domain2", then +username, username@domain1 and username@domain2 will be accepted as valid login +name in addition to just username. + +If used without 'perdomain', domain part will be removed from login before +check with underlying auth. mechanism. If 'perdomain' is set, then +domains must be also set and domain part **will not** be removed before check. + diff --git a/docs/reference/auth/ldap.md b/docs/reference/auth/ldap.md new file mode 100644 index 0000000..a4ced55 --- /dev/null +++ b/docs/reference/auth/ldap.md @@ -0,0 +1,130 @@ +# LDAP BindDN + +maddy supports authentication via LDAP using DN binding. Passwords are verified +by the LDAP server. + +maddy needs to know the DN to use for binding. It can be obtained either by +directory search or template . + +Note that storage backends conventionally use email addresses, if you use +non-email identifiers as usernames then you should map them onto +emails on delivery by using `auth_map` (see documentation page for used storage backend). + +auth.ldap also can be a used as a table module. This way you can check +whether the account exists. It works only if DN template is not used. + +``` +auth.ldap { + urls ldap://maddy.test:389 + + # Specify initial bind credentials. Not required ('bind off') + # if DN template is used. + bind plain "cn=maddy,ou=people,dc=maddy,dc=test" "123456" + + # Specify DN template to skip lookup. + dn_template "cn={username},ou=people,dc=maddy,dc=test" + + # Specify base_dn and filter to lookup DN. + base_dn "ou=people,dc=maddy,dc=test" + filter "(&(objectClass=posixAccount)(uid={username}))" + + tls_client { ... } + starttls off + debug off + connect_timeout 1m +} +``` +``` +auth.ldap ldap://maddy.test.389 { + ... +} +``` + +## Configuration directives + +### urls _servers..._ + +**Required.** + +URLs of the directory servers to use. First available server +is used - no load-balancing is done. + +URLs should use `ldap://`, `ldaps://`, `ldapi://` schemes. + +--- + +### bind `off` | `unauth` | `external` | `plain` _username_ _password_ + +Default: `off` + +Credentials to use for initial binding. Required if DN lookup is used. + +`unauth` performs unauthenticated bind. `external` performs external binding +which is useful for Unix socket connections (`ldapi://`) or TLS client certificate +authentication (cert. is set using tls_client directive). `plain` performs a +simple bind using provided credentials. + +--- + +### dn_template _template_ + +DN template to use for binding. `{username}` is replaced with the +username specified by the user. + +--- + +### base_dn _dn_ + +Base DN to use for lookup. + +--- + +### filter _str_ + +DN lookup filter. `{username}` is replaced with the username specified +by the user. + +Example: + +``` +(&(objectClass=posixAccount)(uid={username})) +``` + +Example (using ActiveDirectory): + +``` +(&(objectCategory=Person)(memberOf=CN=user-group,OU=example,DC=example,DC=org)(sAMAccountName={username})(!(UserAccountControl:1.2.840.113556.1.4.803:=2))) +``` + +Example: + +``` +(&(objectClass=Person)(mail={username})) +``` + +--- + +### starttls _bool_ +Default: `off` + +Whether to upgrade connection to TLS using STARTTLS. + +--- + +### tls_client { ... } + +Advanced TLS client configuration. See [TLS configuration / Client](/reference/tls/#client) for details. + +--- + +### connect_timeout _duration_ +Default: `1m` + +Timeout for initial connection to the directory server. + +--- + +### request_timeout _duration_ +Default: `1m` + +Timeout for each request (binding, lookup). diff --git a/docs/reference/auth/netauth.md b/docs/reference/auth/netauth.md new file mode 100644 index 0000000..2664d41 --- /dev/null +++ b/docs/reference/auth/netauth.md @@ -0,0 +1,48 @@ +# Native NetAuth + +maddy supports authentication via NetAuth using direct entity +authentication checks. Passwords are verified by the NetAuth server. + +maddy needs to know the Entity ID to use for authentication. It must +match the string the user provides for the Local Atom part of their +mail address. + +Note that storage backends conventionally use email addresses. Since +NetAuth recommends *nix compatible usernames, you will need to map the +email identifiers to NetAuth Entity IDs using `auth_map` (see +documentation page for used storage backend). + +auth.netauth also can be used as a table module. This way you can +check whether the account exists. + +Note that the configuration fragment provided below is very sparse. +This is because NetAuth expects to read most of its common +configuration values from the system NetAuth config file located at +`/etc/netauth/config.toml`. + +``` +auth.netauth { + require_group "maddy-users" + debug off +} +``` + +``` +auth.netauth {} +``` + +## Configuration directives + +### require_group _group_ + +Optional. + +Group that entities must possess to be able to use maddy services. +This can be used to provide email to just a subset of the entities +present in NetAuth. + +--- + +### debug `on` | `off` + +Default: `off` diff --git a/docs/reference/auth/pam.md b/docs/reference/auth/pam.md new file mode 100644 index 0000000..89f0f3e --- /dev/null +++ b/docs/reference/auth/pam.md @@ -0,0 +1,48 @@ +# PAM + +auth.pam module implements authentication using libpam. Alternatively it can be configured to +use helper binary like auth.external module does. + +maddy should be built with libpam build tag to use this module without +'use_helper' directive. + +``` +go get -tags 'libpam' ... +``` + +``` +auth.pam { + debug no + use_helper no +} +``` + +## Configuration directives + +### debug _boolean_ +Default: `no` + +Enable verbose logging for all modules. You don't need that unless you are +reporting a bug. + +--- + +### use_helper _boolean_ +Default: `no` + +Use `LibexecDirectory/maddy-pam-helper` instead of directly calling libpam. +You need to use that if: + +1. maddy is not compiled with libpam, but `maddy-pam-helper` is built separately. +2. maddy is running as an unprivileged user and used PAM configuration requires additional privileges (e.g. when using system accounts). + +For 2, you need to make `maddy-pam-helper` binary setuid, see +README.md in source tree for details. + +TL;DR (assuming you have the maddy group): + +``` +chown root:maddy /usr/lib/maddy/maddy-pam-helper +chmod u+xs,g+x,o-x /usr/lib/maddy/maddy-pam-helper +``` + diff --git a/docs/reference/auth/pass_table.md b/docs/reference/auth/pass_table.md new file mode 100644 index 0000000..39fea6f --- /dev/null +++ b/docs/reference/auth/pass_table.md @@ -0,0 +1,44 @@ +# Password table + +auth.pass_table module implements username:password authentication by looking up the +password hash using a table module (maddy-tables(5)). It can be used +to load user credentials from text file (via table.file module) or SQL query +(via table.sql_table module). + + +Definition: +``` +auth.pass_table [block name] { + table + +} +``` +Shortened variant for inline use: +``` +pass_table
[table arguments] { + [additional table config] +} +``` + +Example, read username:password pair from the text file: +``` +smtp tcp://0.0.0.0:587 { + auth pass_table file /etc/maddy/smtp_passwd + ... +} +``` + +## Password hashes + +pass_table expects the used table to contain certain structured values with +hash algorithm name, salt and other necessary parameters. + +You should use `maddy hash` command to generate suitable values. +See `maddy hash --help` for details. + +## maddy creds + +If the underlying table is a "mutable" table (see maddy-tables(5)) then +the `maddy creds` command can be used to modify the underlying tables +via pass_table module. It will act on a "local credentials store" and will write +appropriate hash values to the table. diff --git a/docs/reference/auth/plain_separate.md b/docs/reference/auth/plain_separate.md new file mode 100644 index 0000000..f5b5766 --- /dev/null +++ b/docs/reference/auth/plain_separate.md @@ -0,0 +1,45 @@ +# Separate username and password lookup + +auth.plain_separate module implements authentication using username:password pairs but can +use zero or more "table modules" (maddy-tables(5)) and one or more +authentication providers to verify credentials. + +``` +auth.plain_separate { + user ... + user ... + ... + pass ... + pass ... + ... +} +``` + +How it works: +- Initial username input is normalized using PRECIS UsernameCaseMapped profile. +- Each table specified with the 'user' directive looked up using normalized + username. If match is not found in any table, authentication fails. +- Each authentication provider specified with the 'pass' directive is tried. + If authentication with all providers fails - an error is returned. + +## Configuration directives + +### user _table-module_ + +Configuration block for any module from maddy-tables(5) can be used here. + +Example: + +``` +user file /etc/maddy/allowed_users +``` + +--- + +### pass _auth-provider_ + +Configuration block for any auth. provider module can be used here, even +'plain_split' itself. + +The used auth. provider must provide username:password pair-based +authentication. diff --git a/docs/reference/auth/shadow.md b/docs/reference/auth/shadow.md new file mode 100644 index 0000000..0fc3e89 --- /dev/null +++ b/docs/reference/auth/shadow.md @@ -0,0 +1,40 @@ +# /etc/shadow + +auth.shadow module implements authentication by reading /etc/shadow. Alternatively it can be +configured to use helper binary like auth.external does. + +``` +auth.shadow { + debug no + use_helper no +} +``` + +## Configuration directives + +### debug _boolean_ + +Default: `no` + +Enable verbose logging for all modules. You don't need that unless you are +reporting a bug. + +--- + +### use_helper _boolean_ +Default: `no` + +Use `LibexecDirectory/maddy-shadow-helper` instead of directly reading `/etc/shadow`. +You need to use that if maddy is running as an unprivileged user +privileges (e.g. when using system accounts). + +You need to make `maddy-shadow-helper` binary setuid, see +cmd/maddy-shadow-helper/README.md in source tree for details. + +TL;DR (assuming you have maddy group): + +``` +chown root:maddy /usr/lib/maddy/maddy-shadow-helper +chmod u+xs,g+x,o-x /usr/lib/maddy/maddy-shadow-helper +``` + diff --git a/docs/reference/blob/fs.md b/docs/reference/blob/fs.md new file mode 100644 index 0000000..ef94b54 --- /dev/null +++ b/docs/reference/blob/fs.md @@ -0,0 +1,23 @@ +# Filesystem + +This module stores message bodies in a file system directory. + +``` +storage.blob.fs { + root +} +``` + +``` +storage.blob.fs +``` + +## Configuration directives + +### root _path_ +Default: not set + +Path to the FS directory. Must be readable and writable by the server process. +If it does not exist - it will be created (parent directory should be writable +for this). Relative paths are interpreted relatively to server state directory. + diff --git a/docs/reference/blob/s3.md b/docs/reference/blob/s3.md new file mode 100644 index 0000000..54b6a4e --- /dev/null +++ b/docs/reference/blob/s3.md @@ -0,0 +1,98 @@ +# Amazon S3 + +storage.blob.s3 module stores messages bodies in a bucket on S3-compatible storage. + +``` +storage.blob.s3 { + endpoint play.min.io + secure yes + access_key "Q3AM3UQ867SPQQA43P2F" + secret_key "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" + bucket maddy-test + + # optional + region eu-central-1 + object_prefix maddy/ + creds access_key +} +``` + +Example: + +``` +storage.imapsql local_mailboxes { + ... + msg_store s3 { + endpoint s3.amazonaws.com + access_key "..." + secret_key "..." + bucket maddy-messages + region us-west-2 + creds access_key + } +} +``` + +## Configuration directives + +### endpoint _address:port_ + +**Required**. + +Root S3 endpoint. e.g. `s3.amazonaws.com` + +--- + +### secure _boolean_ +Default: `yes` + +Whether TLS should be used. + +--- + +### access_key _string_
secret_key _string_ + +**Required**. + +Static S3 credentials. + +--- + +### bucket _name_ + +**Required**. + +S3 bucket name. The bucket must exist and +be read-writable. + +--- + +### region _string_ +Default: not set + +S3 bucket location. May be called "endpoint" in some manuals. + +--- + +### object_prefix _string_ +Default: empty string + +String to add to all keys stored by maddy. + +Can be useful when S3 is used as a file system. + +--- + +### creds `access_key` | `file_minio` | `file_aws` | `iam` +Default: `access_key` + +Credentials to use for accessing the S3 Bucket. + +Credential Types: + + - `access_key`: use AWS access key and secret access key + - `file_minio`: use credentials for Minio present at ~/.mc/config.json + - `file_aws`: use credentials for AWS S3 present at ~/.aws/credentials + - `iam`: use AWS IAM instance profile for credentials. + +By default, access_key is used with the access key and secret access key present in the config. diff --git a/docs/reference/checks/actions.md b/docs/reference/checks/actions.md new file mode 100644 index 0000000..d9e9f9c --- /dev/null +++ b/docs/reference/checks/actions.md @@ -0,0 +1,21 @@ +# Check actions + +When a certain check module thinks the message is "bad", it takes some actions +depending on its configuration. Most checks follow the same configuration +structure and allow following actions to be taken on check failure: + +- Do nothing (`action ignore`) + +Useful for testing deployment of new checks. Check failures are still logged +but they have no effect on message delivery. + +- Reject the message (`action reject`) + +Reject the message at connection time. No bounce is generated locally. + +- Quarantine the message (`action quarantine`) + +Mark message as 'quarantined'. If message is then delivered to the local +storage, the storage backend can place the message in the 'Junk' mailbox. +Another thing to keep in mind that 'target.remote' module +will refuse to send quarantined messages. \ No newline at end of file diff --git a/docs/reference/checks/authorize_sender.md b/docs/reference/checks/authorize_sender.md new file mode 100644 index 0000000..4ddd786 --- /dev/null +++ b/docs/reference/checks/authorize_sender.md @@ -0,0 +1,132 @@ +# MAIL FROM and From authorization + +Module check.authorize_sender verifies that envelope and header sender addresses belong +to the authenticated user. Address ownership is established via table +that maps each user account to a email address it is allowed to use. +There are some special cases, see `user_to_email` description below. + +``` +check.authorize_sender { + prepare_email identity + user_to_email identity + check_header yes + + unauth_action reject + no_match_action reject + malformed_action reject + err_action reject + + auth_normalize auto + from_normalize auto +} +``` +``` +check { + authorize_sender { ... } +} +``` + +## Configuration directives + +### user_to_email _table_ +Default: `identity` + +Table that maps authorization username to the list of sender emails +the user is allowed to use. + +In additional to email addresses, the table can contain domain names or +special string "\*" as a value. If the value is a domain - user +will be allowed to use any mailbox within it as a sender address. +If it is "\*" - user will be allowed to use any address. + +By default, table.identity is used, meaning that username should +be equal to the sender email. + +Before username is looked up via the table, normalization algorithm +defined by auth_normalize is applied to it. + +--- + +### prepare_email _table_ +Default: `identity` + +Table that is used to translate email addresses before they +are matched against user_to_email values. + +Typically used to allow users to use their aliases as sender +addresses - prepare_email in this case should translate +aliases to "canonical" addresses. This is how it is +done in default configuration. + +If table does not contain any mapping for the used sender +address, it will be used as is. + +--- + +### check_header _boolean_ +Default: `yes` + +Whether to verify header sender in addition to envelope. + +Either Sender or From field value should match the +authorization identity. + +--- + +### unauth_action _action_ +Default: `reject` + +What to do if the user is not authenticated at all. + +--- + +### no_match_action _action_ +Default: `reject` + +What to do if user is not allowed to use the sender address specified. + +--- + +### malformed_action _action_ +Default: `reject` + +What to do if From or Sender header fields contain malformed values. + +--- + +### err_action _action_ +Default: `reject` + +What to do if error happens during prepare_email or user_to_email lookup. + +--- + +### auth_normalize _action_ +Default: `auto` + +Normalization function to apply to authorization username before +further processing. + +Available options: + +- `auto` `precis_casefold_email` for valid emails, `precis_casefold` otherwise. +- `precis_casefold_email` PRECIS UsernameCaseMapped profile + U-labels form for domain +- `precis_casefold` PRECIS UsernameCaseMapped profile for the entire string +- `precis_email` PRECIS UsernameCasePreserved profile + U-labels form for domain +- `precis` PRECIS UsernameCasePreserved profile for the entire string +- `casefold` Convert to lower case +- `noop` Nothing + +PRECIS profiles are defined by RFC 8265. In short, they make sure +that Unicode strings that look the same will be compared as if they were +the same. CaseMapped profiles also convert strings to lower case. + +--- + +### from_normalize _action_ +Default: `auto` + +Normalization function to apply to email addresses before +further processing. + +Available options are same as for `auth_normalize`. diff --git a/docs/reference/checks/command.md b/docs/reference/checks/command.md new file mode 100644 index 0000000..6475efc --- /dev/null +++ b/docs/reference/checks/command.md @@ -0,0 +1,96 @@ +# System command filter + +This module executes an arbitrary system command during a specified stage of +checks execution. + +``` +command executable_name arg0 arg1 ... { + run_on body + + code 1 reject + code 2 quarantine +} +``` + +## Arguments + +The module arguments specify the command to run. If the first argument is not +an absolute path, it is looked up in the Libexec Directory (/usr/lib/maddy on +Linux) and in $PATH (in that ordering). Note that no additional handling +of arguments is done, especially, the command is executed directly, not via the +system shell. + +There is a set of special strings that are replaced with the corresponding +message-specific values: + +- `{source_ip}` – IPv4/IPv6 address of the sending MTA. +- `{source_host}` – Hostname of the sending MTA, from the HELO/EHLO command. +- `{source_rdns}` – PTR record of the sending MTA IP address. +- `{msg_id}` – Internal message identifier. Unique for each delivery. +- `{auth_user}` – Client username, if authenticated using SASL PLAIN +- `{sender}` – Message sender address, as specified in the MAIL FROM SMTP command. +- `{rcpts}` – List of accepted recipient addresses, including the currently handled + one. +- `{address}` – Currently handled address. This is a recipient address if the command + is called during RCPT TO command handling (`run_on rcpt`) or a sender + address if the command is called during MAIL FROM command handling (`run_on + sender`). + +If value is undefined (e.g. `{source_ip}` for a message accepted over a Unix +socket) or unavailable (the command is executed too early), the placeholder +is replaced with an empty string. Note that it can not remove the argument. +E.g. `-i {source_ip}` will not become just `-i`, it will be `-i ""` + +Undefined placeholders are not replaced. + +## Command stdout + +The command stdout must be either empty or contain a valid RFC 5322 header. +If it contains a byte stream that does not look a valid header, the message +will be rejected with a temporary error. + +The header from stdout will be **prepended** to the message header. + +## Configuration directives + +### run_on `conn` | `sender` | `rcpt` | `body` +Default: `body` + +When to run the command. This directive also affects the information visible +for the message. + +- `conn`
+ Run before the sender address (MAIL FROM) is handled.
+ **Stdin**: Empty
+ **Available placeholders**: {source_ip}, {source_host}, {msg_id}, {auth_user}. + +- `sender`
+ Run during sender address (MAIL FROM) handling.
+ **Stdin**: Empty
+ **Available placeholders**: conn placeholders + {sender}, {address}. + The {address} placeholder contains the MAIL FROM address. + +- `rcpt`
+ Run during recipient address (RCPT TO) handling. The command is executed + once for each RCPT TO command, even if the same recipient is specified + multiple times.
+ **Stdin**: Empty
+ **Available placeholders**: sender placeholders + {rcpts}. + The {address} placeholder contains the recipient address. + +- `body`
+ Run during message body handling.
+ **Stdin**: The message header + body
+ **Available placeholders**: all except for {address}. + +--- + +### code _integer_ ignore
code _integer_ quarantine
code _integer_ reject _smtp-code_ _smtp-enhanced-code_ _smtp-message_ + +This directive specifies the mapping from the command exit code _integer_ to +the message pipeline action. + +Two codes are defined implicitly, exit code 1 causes the message to be rejected +with a permanent error, exit code 2 causes the message to be quarantined. Both +actions can be overridden using the 'code' directive. + diff --git a/docs/reference/checks/dkim.md b/docs/reference/checks/dkim.md new file mode 100644 index 0000000..7ab14a6 --- /dev/null +++ b/docs/reference/checks/dkim.md @@ -0,0 +1,63 @@ +# DKIM + +This is the check module that performs verification of the DKIM signatures +present on the incoming messages. + +## Configuration directives + +``` +check.dkim { + debug no + required_fields From Subject + allow_body_subset no + no_sig_action ignore + broken_sig_action ignore + fail_open no +} +``` + +### debug _boolean_ +Default: global directive value + +Log both successful and unsuccessful check executions instead of just +unsuccessful. + +--- + +### required_fields _string..._ +Default: `From Subject` + +Header fields that should be included in each signature. If signature +lacks any field listed in that directive, it will be considered invalid. + +Note that From is always required to be signed, even if it is not included in +this directive. + +--- + +### no_sig_action _action_ +Default: `ignore` (recommended by RFC 6376) + +Action to take when message without any signature is received. + +Note that DMARC policy of the sender domain can request more strict handling of +missing DKIM signatures. + +--- + +### broken_sig_action _action_ +Default: `ignore` (recommended by RFC 6376) + +Action to take when there are not valid signatures in a message. + +Note that DMARC policy of the sender domain can request more strict handling of +broken DKIM signatures. + +--- + +### fail_open _boolean_ +Default: `no` + +Whether to accept the message if a temporary error occurs during DKIM +verification. Rejecting the message with a 4xx code will require the sender +to resend it later in a hope that the problem will be resolved. diff --git a/docs/reference/checks/dnsbl.md b/docs/reference/checks/dnsbl.md new file mode 100644 index 0000000..a2d2736 --- /dev/null +++ b/docs/reference/checks/dnsbl.md @@ -0,0 +1,173 @@ +# DNSBL lookup + +The check.dnsbl module implements checking of source IP and hostnames against a set +of DNS-based Blackhole lists (DNSBLs). + +Its configuration consists of module configuration directives and a set +of blocks specifying lists to use and kind of lookups to perform on them. + +``` +check.dnsbl { + debug no + check_early no + + quarantine_threshold 1 + reject_threshold 1 + + # Lists configuration example. + dnsbl.example.org { + client_ipv4 yes + client_ipv6 no + ehlo no + mailfrom no + score 1 + } + hsrbl.example.org { + client_ipv4 no + client_ipv6 no + ehlo yes + mailfrom yes + score 1 + } +} +``` + +## Arguments + +Arguments specify the list of IP-based BLs to use. + +The following configurations are equivalent. + +``` +check { + dnsbl dnsbl.example.org dnsbl2.example.org +} +``` + +``` +check { + dnsbl { + dnsbl.example.org dnsbl2.example.org { + client_ipv4 yes + client_ipv6 no + ehlo no + mailfrom no + score 1 + } + } +} +``` + +## Configuration directives + +### debug _boolean_ +Default: global directive value + +Enable verbose logging. + +--- + +### check_early _boolean_ +Default: `no` + +Check BLs before mail delivery starts and silently reject blacklisted clients. + +For this to work correctly, check should not be used in source/destination +pipeline block. + +In particular, this means: + +- No logging is done for rejected messages. +- No action is taken if `quarantine_threshold` is hit, only `reject_threshold` + applies. +- `defer_sender_reject` from SMTP configuration takes no effect. +- MAIL FROM is not checked, even if specified. + +If you often get hit by spam attacks, it is recommended to enable this +setting to save server resources. + +--- + +### quarantine_threshold _integer_ +Default: `1` + +DNSBL score needed (equals-or-higher) to quarantine the message. + +--- + +### reject_threshold _integer_ +Default: `9999` + +DNSBL score needed (equals-or-higher) to reject the message. + +## List configuration + +``` +dnsbl.example.org dnsbl.example.com { + client_ipv4 yes + client_ipv6 no + ehlo no + mailfrom no + responses 127.0.0.1/24 + score 1 +} +``` + +Directive name and arguments specify the actual DNS zone to query when checking +the list. Using multiple arguments is equivalent to specifying the same +configuration separately for each list. + +### client_ipv4 _boolean_ +Default: `yes` + +Whether to check address of the IPv4 clients against the list. + +--- + +### client_ipv6 _boolean_ +Default: `yes` + +Whether to check address of the IPv6 clients against the list. + +--- + +### ehlo _boolean_ +Default: `no` + +Whether to check hostname specified n the HELO/EHLO command +against the list. + +This works correctly only with domain-based DNSBLs. + +--- + +### mailfrom _boolean_ +Default: `no` + +Whether to check domain part of the MAIL FROM address against the list. + +This works correctly only with domain-based DNSBLs. + +--- + +### responses _cidr_ | _ip..._ +Default: `127.0.0.1/24` + +IP networks (in CIDR notation) or addresses to permit in list lookup results. +Addresses not matching any entry in this directives will be ignored. + +--- + +### score _integer_ +Default: `1` + +Score value to add for the message if it is listed. + +If sum of list scores is equals or higher than `quarantine_threshold`, the +message will be quarantined. + +If sum of list scores is equals or higher than `rejected_threshold`, the message +will be rejected. + +It is possible to specify a negative value to make list act like a whitelist +and override results of other blocklists. diff --git a/docs/reference/checks/milter.md b/docs/reference/checks/milter.md new file mode 100644 index 0000000..8286a79 --- /dev/null +++ b/docs/reference/checks/milter.md @@ -0,0 +1,49 @@ +# Milter client + +The 'milter' implements subset of Sendmail's milter protocol that can be used +to integrate external software with maddy. +maddy implements version 6 of the protocol, older versions are +not supported. + +Notable limitations of protocol implementation in maddy include: +1. Changes of envelope sender address are not supported +2. Removal and addition of envelope recipients is not supported +3. Removal and replacement of header fields is not supported +4. Headers fields can be inserted only on top +5. Milter does not receive some "macros" provided by sendmail. + +Restrictions 1 and 2 are inherent to the maddy checks interface and cannot be +removed without major changes to it. Restrictions 3, 4 and 5 are temporary due to +incomplete implementation. + +``` +check.milter { + endpoint + fail_open false +} + +milter +``` + +## Arguments + +When defined inline, the first argument specifies endpoint to access milter +via. See below. + +## Configuration directives + +### endpoint _scheme://path_ +Default: not set + +Specifies milter protocol endpoint to use. +The endpoit is specified in standard URL-like format: +`tcp://127.0.0.1:6669` or `unix:///var/lib/milter/filter.sock` + +--- + +### fail_open _boolean_ +Default: `false` + +Toggles behavior on milter I/O errors. If false ("fail closed") - message is +rejected with temporary error code. If true ("fail open") - check is skipped. + diff --git a/docs/reference/checks/misc.md b/docs/reference/checks/misc.md new file mode 100644 index 0000000..19c71ad --- /dev/null +++ b/docs/reference/checks/misc.md @@ -0,0 +1,48 @@ +# Misc checks + +## Configuration directives + +Following directives are defined for all modules listed below. + +### fail_action `ignore` | `reject` | `quarantine` +Default: `quarantine` + +Action to take when check fails. See [Check actions](../actions/) for details. + +--- + +### debug _boolean_ +Default: global directive value + +Log both successful and unsuccessful check executions instead of just +unsuccessful. + +--- + +### require_mx_record + +Check that domain in MAIL FROM command does have a MX record and none of them +are "null" (contain a single dot as the host). + +By default, quarantines messages coming from servers missing MX records, +use `fail_action` directive to change that. + +--- + +### require_matching_rdns + +Check that source server IP does have a PTR record point to the domain +specified in EHLO/HELO command. + +By default, quarantines messages coming from servers with mismatched or missing +PTR record, use `fail_action` directive to change that. + +--- + +### require_tls + +Check that the source server is connected via TLS; either directly, or by using +the STARTTLS command. + +By default, rejects messages coming from unencrypted servers. Use the +`fail_action` directive to change that. \ No newline at end of file diff --git a/docs/reference/checks/rspamd.md b/docs/reference/checks/rspamd.md new file mode 100644 index 0000000..90063ae --- /dev/null +++ b/docs/reference/checks/rspamd.md @@ -0,0 +1,97 @@ +# rspamd + +The 'rspamd' module implements message filtering by contacting the rspamd +server via HTTP API. + +``` +check.rspamd { + tls_client { ... } + api_path http://127.0.0.1:11333 + settings_id whatever + tag maddy + hostname mx.example.org + io_error_action ignore + error_resp_action ignore + add_header_action quarantine + rewrite_subj_action quarantine + flags pass_all +} + +rspamd http://127.0.0.1:11333 +``` + +## Configuration directives + +### tls_client { ... } +Default: not set + +Configure TLS client if HTTPS is used. See [TLS configuration / Client](/reference/tls/#client) for details. + +--- + +### api_path _url_ +Default: `http://127.0.0.1:11333` + +URL of HTTP API endpoint. Supports both HTTP and HTTPS and can include +path element. + +--- + +### settings_id _string_ +Default: not set + +Settings ID to pass to the server. + +--- + +### tag _string_ +Default: `maddy` + +Value to send in MTA-Tag header field. + +--- + +### hostname _string_
+Default: value of global directive + +Value to send in MTA-Name header field. + +--- + +### io_error_action _action_ +Default: `ignore` + +Action to take in case of inability to contact the rspamd server. + +--- + +### error_resp_action _action_ +Default: `ignore` + +Action to take in case of 5xx or 4xx response received from the rspamd server. + +--- + +### add_header_action _action_ +Default: `quarantine` + +Action to take when rspamd requests to "add header". + +X-Spam-Flag and X-Spam-Score are added to the header irregardless of value. + +--- + +### rewrite_subj_action _action_ +Default: `quarantine` + +Action to take when rspamd requests to "rewrite subject". + +X-Spam-Flag and X-Spam-Score are added to the header irregardless of value. + +--- + +### flags _string-list..._ +Default: `pass_all` + +Flags to pass to the rspamd server. +See [https://rspamd.com/doc/architecture/protocol.html](https://rspamd.com/doc/architecture/protocol.html) for details. diff --git a/docs/reference/checks/spf.md b/docs/reference/checks/spf.md new file mode 100644 index 0000000..f0afb34 --- /dev/null +++ b/docs/reference/checks/spf.md @@ -0,0 +1,97 @@ +# SPF + +check.spf the check module that verifies whether IP address of the client is +authorized to send messages for domain in MAIL FROM address. + +SPF statuses are mapped to maddy check actions in a way +specified by \*_action directives. By default, SPF failure +results in the message being quarantined and errors (both permanent and +temporary) cause message to be rejected. +Authentication-Results field is generated irregardless of status. + +## DMARC override + +It is recommended by the DMARC standard to don't fail delivery based solely on +SPF policy and always check DMARC policy and take action based on it. + +If `enforce_early` is `no`, check.spf module will not take any action on SPF +policy failure if sender domain does have a DMARC record with 'quarantine' or +'reject' policy. Instead it will rely on DMARC support to take necesary +actions using SPF results as an input. + +Disabling `enforce_early` without enabling DMARC support will make SPF policies +no-op and is considered insecure. + +## Configuration directives + +``` +check.spf { + debug no + enforce_early no + fail_action quarantine + softfail_action ignore + permerr_action reject + temperr_action reject +} +``` + +### debug _boolean_ +Default: global directive value + +Enable verbose logging for check.spf. + +--- + +### enforce_early _boolean_ +Default: `no` + +Make policy decision on MAIL FROM stage (before the message body is received). +This makes it impossible to apply DMARC override (see above). + +--- + +### none_action `reject` | `quarantine` | `ignore` +Default: `ignore` + +Action to take when SPF policy evaluates to a 'none' result. + +See [https://tools.ietf.org/html/rfc7208#section-2.6](https://tools.ietf.org/html/rfc7208#section-2.6) for meaning of +SPF results. + +--- + +### neutral_action `reject` | `quarantine` | `ignore` +Default: `ignore` + +Action to take when SPF policy evaluates to a 'neutral' result. + +See [https://tools.ietf.org/html/rfc7208#section-2.6](https://tools.ietf.org/html/rfc7208#section-2.6) for meaning of +SPF results. + +--- + +### fail_action `reject` | `quarantine` | `ignore` +Default: `quarantine` + +Action to take when SPF policy evaluates to a 'fail' result. + +--- + +### softfail_action `reject` | `quarantine` | `ignore` +Default: `ignore` + +Action to take when SPF policy evaluates to a 'softfail' result. + +--- + +### permerr_action `reject` | `quarantine` | `ignore` +Default: `reject` + +Action to take when SPF policy evaluates to a 'permerror' result. + +--- + +### temperr_action `reject` | `quarantine` | `ignore` +Default: `reject` + +Action to take when SPF policy evaluates to a 'temperror' result. diff --git a/docs/reference/config-syntax.md b/docs/reference/config-syntax.md new file mode 100644 index 0000000..72e18d4 --- /dev/null +++ b/docs/reference/config-syntax.md @@ -0,0 +1,200 @@ +# Configuration files syntax + +**Note:** This file is a technical document describing how +maddy parses configuration files. + +Configuration consists of newline-delimited "directives". Each directive can +have zero or more arguments. + +``` +directive0 +directive1 arg0 arg1 +``` + +Any line starting with # is ignored. Empty lines are ignored too. + +## Quoting + +Strings with whitespace should be wrapped into double quotes to make sure they +will be interpreted as a single argument. + +``` +directive0 two arguments +directive1 "one argument" +``` + +String wrapped in quotes may contain newlines and they will not be interpreted +as a directive separator. + +``` +directive0 "one long big +argument for directive0" +``` + +Quotes and only quotes can be escaped inside literals: \\" + +Backslash can be used at the end of line to continue the directve on the next +line. + +## Blocks + +A directive may have several subdirectives. They are written in a {-enclosed +block like this: +``` +directive0 arg0 arg1 { + subdirective0 arg0 arg1 + subdirective1 etc +} +``` + +Subdirectives can have blocks too. + +``` +directive0 { + subdirective0 { + subdirective2 { + a + b + c + } + } + subdirective1 { } +} +``` + +Level of nesting is limited, but you should never hit the limit with correct +configuration. + +In most cases, an empty block is equivalent to no block: +``` +directive { } +directive2 # same as above +``` + +## Environment variables + +Environment variables can be referenced in the configuration using either +{env:VARIABLENAME} syntax. + +Non-existent variables are expanded to empty strings and not removed from +the arguments list. In the following example, directive0 will have one argument +independently of whether VAR is defined. + +``` +directive0 {env:VAR} +``` + +Parse is forgiving and incomplete variable placeholder (e.g. '{env:VAR') will +be left as-is. Variables are expanded inside quotes too. + +## Snippets & imports + +You can reuse blocks of configuration by defining them as "snippets". Snippet +is just a directive with a block, declared tp top level (not inside any blocks) +and with a directive name wrapped in curly braces. + +``` +(snippetname) { + a + b + c +} +``` + +The snippet can then be referenced using 'import' meta-directive. + +``` +unrelated0 +unrelated1 +import snippetname +``` + +The above example will be expanded into the following configuration: + +``` +unrelated0 +unrelated1 +a +b +c +``` + +Import statement also can be used to include content from other files. It works +exactly the same way as with snippets but the file path should be used instead. +The path can be either relative to the location of the currently processed +configuration file or absolute. If there are both snippet and file with the +same name - snippet will be used. + +``` +# /etc/maddy/tls.conf +tls long_path_to_certificate long_path_to_private_key + +# /etc/maddy/maddy.conf +smtp tcp://0.0.0.0:25 { + import tls.conf +} +``` + +``` +# Expanded into: +smtp tcp://0.0.0.0:25 { + tls long_path_to_certificate long_path_to_private_key +} +``` + +The imported file can introduce new snippets and they can be referenced in any +processed configuration file. + +## Duration values + +Directives that accept duration use the following format: A sequence of decimal +digits with an optional fraction and unit suffix (zero can be specified without +a suffix). If multiple values are specified, they will be added. + +Valid unit suffixes: "h" (hours), "m" (minutes), "s" (seconds), "ms" (milliseconds). +Implementation also accepts us and ns for microseconds and nanoseconds, but these +values are useless in practice. + +Examples: +``` +1h +1h 5m +1h5m +0 +``` + +## Data size values + +Similar to duration values, but fractions are not allowed and suffixes are different. + +Valid unit suffixes: "G" (gibibyte, 1024^3 bytes), "M" (mebibyte, 1024^2 bytes), +"K" (kibibyte, 1024 bytes), "B" or "b" (byte). + +Examples: +``` +32M +3M 5K +5b +``` + +Also note that the following is not valid, unlike Duration values syntax: +``` +32M5K +``` + +## Address Definitions + +Maddy configuration uses URL-like syntax to specify network addresses. + +- `unix://file_path` – Unix domain socket. Relative paths are relative to runtime directory (`/run/maddy`). +- `tcp://ADDRESS:PORT` – TCP/IP socket. +- `tls://ADDRESS:PORT` – TCP/IP socket using TLS. + +## Dummy Module + +No-op module. It doesn't need to be configured explicitly and can be referenced +using "dummy" name. It can act as a delivery target or auth. +provider. In the latter case, it will accept any credentials, allowing any +client to authenticate using any username and password (use with care!). + + diff --git a/docs/reference/endpoints/imap.md b/docs/reference/endpoints/imap.md new file mode 100644 index 0000000..ec0d2e6 --- /dev/null +++ b/docs/reference/endpoints/imap.md @@ -0,0 +1,164 @@ +# IMAP4rev1 endpoint + +Module 'imap' is a listener that implements IMAP4rev1 protocol and provides +access to local messages storage specified by 'storage' directive. + +In most cases, local storage modules will auto-create accounts when they are +accessed via IMAP. This relies on authentication provider used by IMAP endpoint +to provide what essentially is access control. There is a caveat, however: this +auto-creation will not happen when delivering incoming messages via SMTP as +there is no authentication to confirm that this account should indeed be +created. + +## Configuration directives + +``` +imap tcp://0.0.0.0:143 tls://0.0.0.0:993 { + tls /etc/ssl/private/cert.pem /etc/ssl/private/pkey.key + io_debug no + debug no + insecure_auth no + sasl_login no + auth pam + storage &local_mailboxes + auth_map identity + auth_map_normalize auto + storage_map identity + storage_map_normalize auto +} +``` + +### tls _certificate-path_ _key-path_ { ... } +Default: global directive value + +TLS certificate & key to use. Fine-tuning of other TLS properties is possible +by specifying a configuration block and options inside it: + +``` +tls cert.crt key.key { + protocols tls1.2 tls1.3 +} +``` + +See [TLS configuration / Server](/reference/tls/#server-side) for details. + +--- + +### proxy_protocol _trusted ips..._ { ... } +Default: not enabled + +Enable use of HAProxy PROXY protocol. Supports both v1 and v2 protocols. +If a list of trusted IP addresses or subnets is provided, only connections +from those will be trusted. + +TLS for the channel between the proxies and maddy can be configured +using a 'tls' directive: +``` +proxy_protocol { + trust 127.0.0.1 ::1 192.168.0.1/24 + tls &proxy_tls +} +``` +Note that the top-level 'tls' directive is not inherited here. If you +need TLS on top of the PROXY protocol, securing the protocol header, +you must declare TLS explicitly. + +--- + +### io_debug _boolean_ +Default: `no` + +Write all commands and responses to stderr. + +--- + +### io_errors _boolean_ +Default: `no` + +Log I/O errors. + +--- + +### debug _boolean_ +Default: global directive value + +Enable verbose logging. + +--- + +### insecure_auth _boolean_ +Default: `no` (`yes` if TLS is disabled) + +Allow plain-text authentication over unencrypted connections. + +--- + +### sasl_login _boolean_ +Default: `no` + +Enable support for SASL LOGIN authentication mechanism used by +some outdated clients. + +--- + +### auth _module-reference_ +**Required.** + +Use the specified module for authentication. + +--- + +### storage _module-reference_ +**Required.** + +Use the specified module for message storage. + +--- + +### storage_map _module-reference_ +Default: `identity` + +Use the specified table to map SASL usernames to storage account names. + +Before username is looked up, it is normalized using function defined by +`storage_map_normalize`. + +This directive is useful if you want users user@example.org and user@example.com +to share the same storage account named "user". In this case, use + +``` + storage_map email_localpart +``` + +Note that `storage_map` does not affect the username passed to the +authentication provider. + +It also does not affect how message delivery is handled, you should specify +`delivery_map` in storage module to define how to map email addresses +to storage accounts. E.g. + +``` + storage.imapsql local_mailboxes { + ... + delivery_map email_localpart # deliver "user@*" to mailbox for "user" + } +``` + +--- + +### storage_map_normalize _function_ +Default: `auto` + +Same as `auth_map_normalize` but for `storage_map`. + +--- + +### auth_map_normalize _function_ +Default: `auto` + +Overrides global `auth_map_normalize` value for this endpoint. + +See [Global configuration](/reference/global-config) for details. + + + diff --git a/docs/reference/endpoints/openmetrics.md b/docs/reference/endpoints/openmetrics.md new file mode 100644 index 0000000..f455f71 --- /dev/null +++ b/docs/reference/endpoints/openmetrics.md @@ -0,0 +1,41 @@ +# OpenMetrics/Prometheus telemetry + +Various server statistics are provided in OpenMetrics format by the +"openmetrics" module. + +To enable it, add the following line to the server config: + +``` +openmetrics tcp://127.0.0.1:9749 { } +``` + +Scrape endpoint would be `http://127.0.0.1:9749/metrics`. + +## Metrics + +``` +# AUTH command failures due to invalid credentials. +maddy_smtp_failed_logins{module} +# Failed SMTP transaction commands (MAIL, RCPT, DATA). +maddy_smtp_failed_commands{module, command, smtp_code, smtp_enchcode} +# Messages rejected with 4xx code due to ratelimiting. +maddy_smtp_ratelimit_deferred{module} +# Amount of started SMTP transactions started. +maddy_smtp_started_transactions{module} +# Amount of aborted SMTP transactions started. +maddy_smtp_aborted_transactions{module} +# Amount of completed SMTP transactions. +maddy_smtp_completed_transactions{module} +# Number of times a check returned 'reject' result (may be more than processed +# messages if check does so on per-recipient basis). +maddy_check_reject{check} +# Number of times a check returned 'quarantine' result (may be more than +# processed messages if check does so on per-recipient basis). +maddy_check_quarantined{check} +# Amount of queued messages. +maddy_queue_length{module, location} +# Outbound connections established with specific TLS security level. +maddy_remote_conns_tls_level{module, level} +# Outbound connections established with specific MX security level. +maddy_remote_conns_mx_level{module, level} +``` diff --git a/docs/reference/endpoints/smtp.md b/docs/reference/endpoints/smtp.md new file mode 100644 index 0000000..4dfa723 --- /dev/null +++ b/docs/reference/endpoints/smtp.md @@ -0,0 +1,312 @@ +# SMTP/LMTP/Submission endpoint + +Module 'smtp' is a listener that implements ESMTP protocol with optional +authentication, LMTP and Submission support. Incoming messages are processed in +accordance with pipeline rules (explained in Message pipeline section below). + +``` +smtp tcp://0.0.0.0:25 { + hostname example.org + tls /etc/ssl/private/cert.pem /etc/ssl/private/pkey.key + io_debug no + debug no + insecure_auth no + sasl_login no + read_timeout 10m + write_timeout 1m + max_message_size 32M + max_header_size 1M + auth pam + defer_sender_reject yes + dmarc yes + smtp_max_line_length 4000 + limits { + endpoint rate 10 + endpoint concurrency 500 + } + + # Example pipeline ocnfiguration. + destination example.org { + deliver_to &local_mailboxes + } + default_destination { + reject + } +} +``` + +## Configuration directives + +### hostname _string_ +Default: global directive value + +Server name to use in SMTP banner. + +``` +220 example.org ESMTP Service Ready +``` + +--- + +### tls _certificate-path_ _key-path_ { ... } +Default: global directive value + +TLS certificate & key to use. Fine-tuning of other TLS properties is possible +by specifying a configuration block and options inside it: + +``` +tls cert.crt key.key { + protocols tls1.2 tls1.3 +} +``` + +See [TLS configuration / Server](/reference/tls/#server-side) for details. + +--- + +### proxy_protocol _trusted ips..._ { ... }
+Default: not enabled + +Enable use of HAProxy PROXY protocol. Supports both v1 and v2 protocols. +If a list of trusted IP addresses or subnets is provided, only connections +from those will be trusted. + +TLS for the channel between the proxies and maddy can be configured +using a 'tls' directive: +``` +proxy_protocol { + trust 127.0.0.1 ::1 192.168.0.1/24 + tls &proxy_tls +} +``` + +--- + +### io_debug _boolean_ +Default: `no` + +Write all commands and responses to stderr. + +--- + +### debug _boolean_ +Default: global directive value + +Enable verbose logging. + +--- + +### insecure_auth _boolean_ +Default: `no` (`yes` if TLS is disabled) + +Allow plain-text authentication over unencrypted connections. Not recommended! + +--- + +### sasl_login _boolean_ +Default: `no` + +Enable support for SASL LOGIN authentication mechanism used by +some outdated clients. + +--- + +### read_timeout _duration_ +Default: `10m` + +I/O read timeout. + +--- + +### write_timeout _duration_ +Default: `1m` + +I/O write timeout. + +--- + +### max_message_size _size_ +Default: `32M` + +Limit the size of incoming messages to 'size'. + +--- + +### max_header_size _size_ +Default: `1M` + +Limit the size of incoming message headers to 'size'. + +--- + +### auth _module-reference_ +Default: not specified + +Use the specified module for authentication. + +--- + +### defer_sender_reject _boolean_ +Default: `yes` + +Apply sender-based checks and routing logic when first RCPT TO command +is received. This allows maddy to log recipient address of the rejected +message and also improves interoperability with (improperly implemented) +clients that don't expect an error early in session. + +--- + +### max_logged_rcpt_errors _integer_ +Default: `5` + +Amount of RCPT-time errors that should be logged. Further errors will be +handled silently. This is to prevent log flooding during email dictionary +attacks (address probing). + +--- + +### max_received _integer_ +Default: `50` + +Max. amount of Received header fields in the message header. If the incoming +message has more fields than this number, it will be rejected with the permanent error +5.4.6 ("Routing loop detected"). + +--- + +### buffer `ram`
buffer `fs` _path_
buffer `auto` _max-size_ _path_ +Default: `auto 1M StateDirectory/buffer` + +Temporary storage to use for the body of accepted messages. + +- `ram` – Store the body in RAM. +- `fs` – Write out the message to the FS and read it back as needed. +_path_ can be omitted and defaults to StateDirectory/buffer. +- `auto` – Store message bodies smaller than `_max_size_` entirely in RAM, +otherwise write them out to the FS. _path_ can be omitted and defaults to `StateDirectory/buffer`. + +--- + +### smtp_max_line_length _integer_ +Default: `4000` + +The maximum line length allowed in the SMTP input stream. If client sends a +longer line - connection will be closed and message (if any) will be rejected +with a permanent error. + +RFC 5321 has the recommended limit of 998 bytes. Servers are not required +to handle longer lines correctly but some senders may produce them. + +Unless BDAT extension is used by the sender, this limitation also applies to +the message body. + +--- + +### dmarc _boolean_ +Default: `yes` + +Enforce sender's DMARC policy. Due to implementation limitations, it is not a +check module. + +**Note**: Report generation is not implemented now. + +**Note**: DMARC needs SPF and DKIM checks to function correctly. +Without these, DMARC check will not run. + +--- + +## Rate & concurrency limiting + +### limits { ... } +Default: no limits + +This allows configuring a set of message flow restrictions including +max. concurrency and rate per-endpoint, per-source, per-destination. + +Limits are specified as directives inside the block: + +``` +limits { + all rate 20 + destination concurrency 5 +} +``` + +Supported limits: + +### _scope_ rate _burst_ _period_ + +Rate limit. Restrict the amount of messages processed in _period_ to +_burst_ messages. If period is not specified, 1 second is used. + +### _scope_ concurrency _max_ +Concurrency limit. Restrict the amount of messages processed in parallel +to _max_. + +For each supported limitation, _scope_ determines whether it should be applied +for all messages ("all"), per-sender IP ("ip"), per-sender domain ("source") or +per-recipient domain ("destination"). Having a scope other than "all" means +that the restriction will be enforced independently for each group determined +by scope. E.g. "ip rate 20" means that the same IP cannot send more than 20 +messages per second. "destination concurrency 5" means that no more than 5 +messages can be sent in parallel to a single domain. + +**Note**: At the moment, SMTP endpoint on its own does not support per-recipient +limits. They will be no-op. If you want to enforce a per-recipient restriction +on outbound messages, do so using 'limits' directive for the 'table.remote' module + +It is possible to share limit counters between multiple endpoints (or any other +modules). To do so define a top-level configuration block for module "limits" +and reference it where needed using standard & syntax. E.g. + +``` +limits inbound_limits { + all rate 20 +} + +smtp smtp://0.0.0.0:25 { + limits &inbound_limits + ... +} + +submission tls://0.0.0.0:465 { + limits &inbound_limits + ... +} +``` + +Using an "all rate" restriction in such way means that no more than 20 +messages can enter the server through both endpoints in one second. + +# Submission module (submission) + +Module 'submission' implements all functionality of the 'smtp' module and adds +certain message preprocessing on top of it, additionally authentication is +always required. + +'submission' module checks whether addresses in header fields From, Sender, To, +Cc, Bcc, Reply-To are correct and adds Message-ID and Date if it is missing. + +``` +submission tcp://0.0.0.0:587 tls://0.0.0.0:465 { + # ... same as smtp ... +} +``` + +# LMTP module (lmtp) + +Module 'lmtp' implements all functionality of the 'smtp' module but uses +LMTP (RFC 2033) protocol. + +``` +lmtp unix://lmtp.sock { + # ... same as smtp ... +} +``` + +## Limitations of LMTP implementation + +- Can't be used with TCP. +- Delivery to 'sql' module storage is always atomic, either all recipients will + succeed or none of them will. + diff --git a/docs/reference/global-config.md b/docs/reference/global-config.md new file mode 100644 index 0000000..db0ec1a --- /dev/null +++ b/docs/reference/global-config.md @@ -0,0 +1,153 @@ +# Global configuration directives + +These directives can be specified outside of any +configuration blocks and they are applied to all modules. + +Some directives can be overridden on per-module basis (e.g. hostname). + +### state_dir _path_ +Default: `/var/lib/maddy` + +The path to the state directory. This directory will be used to store all +persistent data and should be writable. + +--- + +### runtime_dir _path_ +Default: `/run/maddy` + +The path to the runtime directory. Used for Unix sockets and other temporary +objects. Should be writable. + +--- + +### hostname _domain_ +Default: not specified + +Internet hostname of this mail server. Typicall FQDN is used. It is recommended +to make sure domain specified here resolved to the public IP of the server. + +--- + +### auth_map _module-reference_ +Default: `identity` + +Use the specified table to translate SASL usernames before passing it to the +authentication provider. + +Before username is looked up, it is normalized using function defined by +`auth_map_normalize`. + +Note that `auth_map` does not affect the storage account name used. You probably +should also use `storage_map` in IMAP config block to handle this. + +This directive is useful if used authentication provider does not support +using emails as usernames but you still want users to have separate mailboxes +on separate domains. In this case, use it with `email_localpart` table: + +``` + auth_map email_localpart +``` + +With this configuration, `user@example.org` and `user@example.com` will use +`user` credentials when authenticating, but will access `user@example.org` and +`user@example.com` mailboxes correspondingly. If you want to also accept +`user` as a username, use `auth_map email_localpart_optional`. + +If you want `user@example.org` and `user@example.com` to have the same mailbox, +also set `storage_map` in IMAP config block to use `email_localpart` +(or `email_localpart_optional` if you want to also accept just "user"): + +``` + storage_map email_localpart +``` + +In this case you will need to create storage accounts without domain part in +the name: + +``` +maddy imap-acct create user # instead of user@example.org +``` + +--- + +### auth_map_normalize _function_ +Default: `auto` + +Normalization function to apply to SASL usernames before mapping +them to storage accounts. + +Available options: + +- `auto` `precis_casefold_email` for valid emails, `precis_casefold` otherwise. +- `precis_casefold_email` PRECIS UsernameCaseMapped profile + U-labels form for domain +- `precis_casefold` PRECIS UsernameCaseMapped profile for the entire string +- `precis_email` PRECIS UsernameCasePreserved profile + U-labels form for domain +- `precis` PRECIS UsernameCasePreserved profile for the entire string +- `casefold` Convert to lower case +- `noop` Nothing + +--- + +### autogenerated_msg_domain _domain_ +Default: not specified + +Domain that is used in From field for auto-generated messages (such as Delivery +Status Notifications). + +--- + +### tls `file` _cert-file_ _pkey-file_ | _module-reference_ | `off` +Default: not specified + +Default TLS certificate to use for all endpoints. + +Must be present in either all endpoint modules configuration blocks or as +global directive. + +You can also specify other configuration options such as cipher suites and TLS +version. See maddy-tls(5) for details. maddy uses reasonable +cipher suites and TLS versions by default so you generally don't have to worry +about it. + +--- + +### tls_client { ... } +Default: not specified + +This is optional block that specifies various TLS-related options to use when +making outbound connections. See TLS client configuration for details on +directives that can be used in it. maddy uses reasonable cipher suites and TLS +versions by default so you generally don't have to worry about it. + +--- + +### log _targets..._ | `off` +Default: `stderr` + +Write log to one of more "targets". + +The target can be one or the following: + +- `stderr` – Write logs to stderr. +- `stderr_ts` – Write logs to stderr with timestamps. +- `syslog` – Send logs to the local syslog daemon. +- _file path_ – Write (append) logs to file. + +Example: + +``` +log syslog /var/log/maddy.log +``` + +**Note:** Maddy does not perform log files rotation, this is the job of the +logrotate daemon. Send SIGUSR1 to maddy process to make it reopen log files. + +--- + +### debug _boolean_ +Default: `no` + +Enable verbose logging for all modules. You don't need that unless you are +reporting a bug. + diff --git a/docs/reference/modifiers/dkim.md b/docs/reference/modifiers/dkim.md new file mode 100644 index 0000000..36fffe2 --- /dev/null +++ b/docs/reference/modifiers/dkim.md @@ -0,0 +1,225 @@ +# DKIM signing + +modify.dkim module is a modifier that signs messages using DKIM +protocol (RFC 6376). + +Each configuration block specifies a single selector +and one or more domains. + +A key will be generated or read for each domain, the key to use +for each message will be selected based on the SMTP envelope sender. Exception +for that is that for domain-less postmaster address and null address, the +key for the first domain will be used. If domain in envelope sender +does not match any of loaded keys, message will not be signed. +Additionally, for each messages From header is checked to +match MAIL FROM and authorization identity (username sender is logged in as). +This can be controlled using require_sender_match directive. + +Generated private keys are stored in unencrypted PKCS#8 format +in state_directory/dkim_keys (`/var/lib/maddy/dkim_keys`). +In the same directory .dns files are generated that contain +public key for each domain formatted in the form of a DNS record. + +## Arguments + +domains and selector can be specified in arguments, so actual modify.dkim use can +be shortened to the following: + +``` +modify { + dkim example.org selector +} +``` + +## Configuration directives + +``` +modify.dkim { + debug no + domains example.org example.com + selector default + key_path dkim-keys/{domain}-{selector}.key + oversign_fields ... + sign_fields ... + header_canon relaxed + body_canon relaxed + sig_expiry 120h # 5 days + hash sha256 + newkey_algo rsa2048 +} +``` + +### debug _boolean_ +Default: global directive value + +Enable verbose logging. + +--- + +### domains _string-list_ +**Required**.
+Default: not specified + + +ADministrative Management Domains (ADMDs) taking responsibility for messages. + +Should be specified either as a directive or as an argument. + +--- + +### selector _string_ +**Required**.
+Default: not specified + +Identifier of used key within the ADMD. +Should be specified either as a directive or as an argument. + +--- + +### key_path _string_ +Default: `dkim_keys/{domain}_{selector}.key` + +Path to private key. It should be in PKCS#8 format wrapped in PAM encoding. +If key does not exist, it will be generated using algorithm specified +in newkey_algo. + +Placeholders '{domain}' and '{selector}' will be replaced with corresponding +values from domain and selector directives. + +Additionally, keys in PKCS#1 ("RSA PRIVATE KEY") and +RFC 5915 ("EC PRIVATE KEY") can be read by modify.dkim. Note, however that +newly generated keys are always in PKCS#8. + +--- + +### oversign_fields _list..._ +Default: see below + +Header fields that should be signed n+1 times where n is times they are +present in the message. This makes it impossible to replace field +value by prepending another field with the same name to the message. + +Fields specified here don't have to be also specified in `sign_fields`. + +Default set of oversigned fields: + +- Subject +- To +- From +- Date +- MIME-Version +- Content-Type +- Content-Transfer-Encoding +- Reply-To +- Message-Id +- References +- Autocrypt +- Openpgp + +--- + +### sign_fields _list..._ +Default: see below + +Header fields that should be signed n times where n is times they are +present in the message. For these fields, additional values can be prepended +by intermediate relays, but existing values can't be changed. + +Default set of signed fields: + +- List-Id +- List-Help +- List-Unsubscribe +- List-Post +- List-Owner +- List-Archive +- Resent-To +- Resent-Sender +- Resent-Message-Id +- Resent-Date +- Resent-From +- Resent-Cc + +--- + +### header_canon `relaxed` | `simple` +Default: `relaxed` + +Canonicalization algorithm to use for header fields. With `relaxed`, whitespace within +fields can be modified without breaking the signature, with `simple` no +modifications are allowed. + +--- + +### body_canon `relaxed` | `simple` +Default: `relaxed` + +Canonicalization algorithm to use for message body. With `relaxed`, whitespace within +can be modified without breaking the signature, with `simple` no +modifications are allowed. + +--- + +### sig_expiry _duration_ +Default: `120h` + +Time for which signature should be considered valid. Mainly used to prevent +unauthorized resending of old messages. + +--- + +### hash _hash_ +Default: `sha256` + +Hash algorithm to use when computing body hash. + +sha256 is the only supported algorithm now. + +--- + +### newkey_algo `rsa4096` | `rsa2048` | `ed25519` +Default: `rsa2048` + +Algorithm to use when generating a new key. + +Currently ed25519 is **not** supported by most platforms. + +--- + +### require_sender_match _ids..._ +Default: `envelope auth` + +Require specified identifiers to match From header field and key domain, +otherwise - don't sign the message. + +If From field contains multiple addresses, message will not be +signed unless `allow_multiple_from` is also specified. In that +case only first address will be compared. + +Matching is done in a case-insensitive way. + +Valid values: + +- `off` – Disable check, always sign. +- `envelope` – Require MAIL FROM address to match From header. +- `auth` – If authorization identity contains @ - then require it to + fully match From header. Otherwise, check only local-part + (username). + +--- + +### allow_multiple_from _boolean_ +Default: `no` + +Allow multiple addresses in From header field for purposes of +`require_sender_match` checks. Only first address will be checked, however. + +--- + +### sign_subdomains _boolean_ +Default: `no` + +Sign emails from subdomains using a top domain key. + +Allows only one domain to be specified (can be worked around by using `modify.dkim` +multiple times). diff --git a/docs/reference/modifiers/envelope.md b/docs/reference/modifiers/envelope.md new file mode 100644 index 0000000..0e101cf --- /dev/null +++ b/docs/reference/modifiers/envelope.md @@ -0,0 +1,63 @@ +# Envelope sender / recipient rewriting + +`replace_sender` and `replace_rcpt` modules replace SMTP envelope addresses +based on the mapping defined by the table module (maddy-tables(5)). It is possible +to specify 1:N mappings. This allows, for example, implementing mailing lists. + +The address is normalized before lookup (Punycode in domain-part is decoded, +Unicode is normalized to NFC, the whole string is case-folded). + +First, the whole address is looked up. If there is no replacement, local-part +of the address is looked up separately and is replaced in the address while +keeping the domain part intact. Replacements are not applied recursively, that +is, lookup is not repeated for the replacement. + +Recipients are not deduplicated after expansion, so message may be delivered +multiple times to a single recipient. However, used delivery target can apply +such deduplication (imapsql storage does it). + +Definition: + +``` +replace_rcpt
[table arguments] { + [extended table config] +} +replace_sender
[table arguments] { + [extended table config] +} +``` + +Use examples: + +``` +modify { + replace_rcpt file /etc/maddy/aliases + replace_rcpt static { + entry a@example.org b@example.org + entry c@example.org c1@example.org c2@example.org + } + replace_rcpt regexp "(.+)@example.net" "$1@example.org" + replace_rcpt regexp "(.+)@example.net" "$1@example.org" "$1@example.com" +} +``` + +Possible contents of /etc/maddy/aliases in the example above: + +``` +# Replace 'cat' with any domain to 'dog'. +# E.g. cat@example.net -> dog@example.net +cat: dog + +# Replace cat@example.org with cat@example.com. +# Takes priority over the previous line. +cat@example.org: cat@example.com + +# Using aliases in multiple lines +cat2: dog +cat2: mouse +cat2@example.org: cat@example.com +cat2@example.org: cat@example.net +# Comma-separated aliases in multiple lines +cat3: dog , mouse +cat3@example.org: cat@example.com , cat@example.net +``` \ No newline at end of file diff --git a/docs/reference/modules.md b/docs/reference/modules.md new file mode 100644 index 0000000..f327e86 --- /dev/null +++ b/docs/reference/modules.md @@ -0,0 +1,76 @@ +# Modules introduction + +maddy is built of many small components called "modules". Each module does one +certain well-defined task. Modules can be connected to each other in arbitrary +ways to achieve wanted functionality. Default configuration file defines +set of modules that together implement typical email server stack. + +To specify the module that should be used by another module for something, look +for configuration directives with "module reference" argument. Then +put the module name as an argument for it. Optionally, if referenced module +needs that, put additional arguments after the name. You can also put a +configuration block with additional directives specifing the module +configuration. + +Here are some examples: + +``` +smtp ... { + # Deliver messages to the 'dummy' module with the default configuration. + deliver_to dummy + + # Deliver messages to the 'target.smtp' module with + # 'tcp://127.0.0.1:1125' argument as a configuration. + deliver_to smtp tcp://127.0.0.1:1125 + + # Deliver messages to the 'queue' module with the specified configuration. + deliver_to queue { + target ... + max_tries 10 + } +} +``` + +Additionally, module configuration can be placed in a separate named block +at the top-level and referenced by its name where it is needed. + +Here is the example: +``` +storage.imapsql local_mailboxes { + driver sqlite3 + dsn all.db +} + +smtp ... { + deliver_to &local_mailboxes +} +``` + +It is recommended to use this syntax for modules that are 'expensive' to +initialize such as storage backends and authentication providers. + +For top-level configuration block definition, syntax is as follows: +``` +namespace.module_name config_block_name... { + module_configuration +} +``` +If config\_block\_name is omitted, it will be the same as module\_name. Multiple +names can be specified. All names must be unique. + +Note the "storage." prefix. This is the actual module name and includes +"namespace". It is a little cheating to make more concise names and can +be omitted when you reference the module where it is used since it can +be implied (e.g. putting module reference in "check{}" likely means you want +something with "check." prefix) + +Usual module arguments can't be specified when using this syntax, however, +modules usually provide explicit directives that allow to specify the needed +values. For example 'sql sqlite3 all.db' is equivalent to +``` +storage.imapsql { + driver sqlite3 + dsn all.db +} +``` + diff --git a/docs/reference/smtp-pipeline.md b/docs/reference/smtp-pipeline.md new file mode 100644 index 0000000..b094343 --- /dev/null +++ b/docs/reference/smtp-pipeline.md @@ -0,0 +1,408 @@ +# SMTP message routing (pipeline) + +# Message pipeline + +A message pipeline is a set of module references and associated rules that +describe how to handle messages. + +The pipeline is responsible for + +- Running message filters (called "checks"), (e.g. DKIM signature verification, + DNSBL lookup, and so on). +- Running message modifiers (e.g. DKIM signature creation). +- Associating each message recipient with one or more delivery targets. + Delivery target is a module that does the final processing (delivery) of the + message. + +Message handling flow is as follows: + +- Execute checks referenced in top-level `check` blocks (if any) +- Execute modifiers referenced in top-level `modify` blocks (if any) +- If there are `source` blocks - select one that matches the message sender (as + specified in MAIL FROM). If there are no `source` blocks - the entire + configuration is assumed to be the `default_source` block. +- Execute checks referenced in `check` blocks inside the selected `source` block + (if any). +- Execute modifiers referenced in `modify` blocks inside selected `source` + block (if any). + +Then, for each recipient: + +- Select the `destination` block that matches it. If there are + no `destination` blocks - the entire used `source` block is interpreted as if it + was a `default_destination` block. +- Execute checks referenced in the `check` block inside the selected `destination` + block (if any). +- Execute modifiers referenced in `modify` block inside the selected `destination` + block (if any). +- If the used block contains the `reject` directive - reject the recipient with + the specified SMTP status code. +- If the used block contains the `deliver_to` directive - pass the message to the + specified target module. Only recipients that are handled + by the used block are visible to the target. + +Each recipient is handled only by a single `destination` block, in case of +overlapping `destination` - the first one takes priority. + +``` +destination example.org { + deliver_to targetA +} +destination example.org { # ambiguous and thus not allowed + deliver_to targetB +} +``` + +Same goes for `source` blocks, each message is handled only by a single block. + +Each recipient block should contain at least one `deliver_to` directive or +`reject` directive. If `destination` blocks are used, then +`default_destination` block should also be used to specify behavior for +unmatched recipients. Same goes for source blocks, `default_source` should be +used if `source` is used. + +That is, pipeline configuration should explicitly specify behavior for each +possible sender/recipient combination. + +Additionally, directives that specify final handling decision (`deliver_to`, +`reject`) can't be used at the same level as source/destination rules. +Consider example: + +``` +destination example.org { + deliver_to local_mboxes +} +reject +``` + +It is not obvious whether `reject` applies to all recipients or +just for non-example.org ones, hence this is not allowed. + +Complete configuration example using all of the mentioned directives: + +``` +check { + # Run a check to make sure source SMTP server identification + # is legit. + spf +} + +# Messages coming from senders at example.org will be handled in +# accordance with the following configuration block. +source example.org { + # We are example.com, so deliver all messages with recipients + # at example.com to our local mailboxes. + destination example.com { + deliver_to &local_mailboxes + } + + # We don't do anything with recipients at different domains + # because we are not an open relay, thus we reject them. + default_destination { + reject 521 5.0.0 "User not local" + } +} + +# We do our business only with example.org, so reject all +# other senders. +default_source { + reject +} +``` + +## Directives + + +### check _block name_ { ... } +Context: pipeline configuration, source block, destination block + +List of the module references for checks that should be executed on +messages handled by block where 'check' is placed in. + +Note that message body checks placed in destination block are currently +ignored. Due to the way SMTP protocol is defined, they would cause message to +be rejected for all recipients which is not what you usually want when using +such configurations. + +Example: + +``` +check { + # Reference implicitly defined default configuration for check. + spf + + # Inline definition of custom config. + spf { + # Configuration for spf goes here. + permerr_action reject + } +} +``` + +It is also possible to define the block of checks at the top level +as "checks" module and reference it using & syntax. Example: + +``` +checks inbound_checks { + spf + dkim +} + +# ... somewhere else ... +{ + ... + check &inbound_checks +} +``` + +--- + +### modify { ... } +Default: not specified
+Context: pipeline configuration, source block, destination block + +List of the module references for modifiers that should be executed on +messages handled by block where 'modify' is placed in. + +Message modifiers are similar to checks with the difference in that checks +purpose is to verify whether the message is legitimate and valid per local +policy, while modifier purpose is to post-process message and its metadata +before final delivery. + +For example, modifier can replace recipient address to make message delivered +to the different mailbox or it can cryptographically sign outgoing message +(e.g. using DKIM). Some modifier can perform multiple unrelated modifications +on the message. + +**Note**: Modifiers that affect source address can be used only globally or on +per-source basis, they will be no-op inside destination blocks. Modifiers that +affect the message header will affect it for all recipients. + +It is also possible to define the block of modifiers at the top level +as "modiifers" module and reference it using & syntax. Example: + +``` +modifiers local_modifiers { + replace_rcpt file /etc/maddy/aliases +} + +# ... somewhere else ... +{ + ... + modify &local_modifiers +} +``` + +--- + +### reject _smtp-code_ _smtp-enhanced-code_ _error-description_
reject _smtp-code_ _smtp-enhanced-code_
reject _smtp-code_
reject +Context: destination block + +Messages handled by the configuration block with this directive will be +rejected with the specified SMTP error. + +If you aren't sure which codes to use, use 541 and 5.4.0 with your message or +just leave all arguments out, the error description will say "message is +rejected due to policy reasons" which is usually what you want to mean. + +`reject` can't be used in the same block with `deliver_to` or +`destination`/`source` directives. + +Example: + +``` +reject 541 5.4.0 "We don't like example.org, go away" +``` + +--- + +### deliver_to _target-config-block_ +Context: pipeline configuration, source block, destination block + +Deliver the message to the referenced delivery target. What happens next is +defined solely by used target. If `deliver_to` is used inside `destination` +block, only matching recipients will be passed to the target. + +--- + +### source_in _table-reference_ { ... } +Context: pipeline configuration + +Handle messages with envelope senders present in the specified table in +accordance with the specified configuration block. + +Takes precedence over all `sender` directives. + +Example: + +``` +source_in file /etc/maddy/banned_addrs { + reject 550 5.7.0 "You are not welcome here" +} +source example.org { + ... +} +... +``` + +See `destination_in` documentation for note about table configuration. + +--- + +### source _rules..._ { ... } +Context: pipeline configuration + +Handle messages with MAIL FROM value (sender address) matching any of the rules +in accordance with the specified configuration block. + +"Rule" is either a domain or a complete address. In case of overlapping +'rules', first one takes priority. Matching is case-insensitive. + +Example: + +``` +# All messages coming from example.org domain will be delivered +# to local_mailboxes. +source example.org { + deliver_to &local_mailboxes +} +# Messages coming from different domains will be rejected. +default_source { + reject 521 5.0.0 "You were not invited" +} +``` + +--- + +### reroute { ... } +Context: pipeline configuration, source block, destination block + +This directive allows to make message routing decisions based on the +result of modifiers. The block can contain all pipeline directives and they +will be handled the same with the exception that source and destination rules +will use the final recipient and sender values (e.g. after all modifiers are +applied). + +Here is the concrete example how it can be useful: + +``` +destination example.org { + modify { + replace_rcpt file /etc/maddy/aliases + } + reroute { + destination example.org { + deliver_to &local_mailboxes + } + default_destination { + deliver_to &remote_queue + } + } +} +``` + +This configuration allows to specify alias local addresses to remote ones +without being an open relay, since remote_queue can be used only if remote +address was introduced as a result of rewrite of local address. + +**Warning**: If you have DMARC enabled (default), results generated by SPF +and DKIM checks inside a reroute block **will not** be considered in DMARC +evaluation. + +--- + +### destination_in _table-reference_ { ... } +Context: pipeline configuration, source block + +Handle messages with envelope recipients present in the specified table in +accordance with the specified configuration block. + +Takes precedence over all 'destination' directives. + +Example: + +``` +destination_in file /etc/maddy/remote_addrs { + deliver_to smtp tcp://10.0.0.7:25 +} +destination example.com { + deliver_to &local_mailboxes +} +... +``` + +Note that due to the syntax restrictions, it is not possible to specify +extended configuration for table module. E.g. this is not valid: + +``` +destination_in sql_table { + dsn ... + driver ... +} { + deliver_to whatever +} +``` + +In this case, configuration should be specified separately and be referneced +using '&' syntax: + +``` +table.sql_table remote_addrs { + dsn ... + driver ... +} + +whatever { + destination_in &remote_addrs { + deliver_to whatever + } +} +``` + +--- + +### destination _rule..._ { ... } +Context: pipeline configuration, source block + +Handle messages with RCPT TO value (recipient address) matching any of the +rules in accordance with the specified configuration block. + +"Rule" is either a domain or a complete address. Duplicate rules are not +allowed. Matching is case-insensitive. + +Note that messages with multiple recipients are split into multiple messages if +they have recipients matched by multiple blocks. Each block will see the +message only with recipients matched by its rules. + +Example: + +``` +# Messages with recipients at example.com domain will be +# delivered to local_mailboxes target. +destination example.com { + deliver_to &local_mailboxes +} + +# Messages with other recipients will be rejected. +default_destination { + rejected 541 5.0.0 "User not local" +} +``` + +## Reusable pipeline snippets (msgpipeline module) + +The message pipeline can be used independently of the SMTP module in other +contexts that require a delivery target via `msgpipeline` module. + +Example: + +``` +msgpipeline local_routing { + destination whatever.com { + deliver_to dummy + } +} + +# ... somewhere else ... +deliver_to &local_routing +``` \ No newline at end of file diff --git a/docs/reference/storage/imap-filters.md b/docs/reference/storage/imap-filters.md new file mode 100644 index 0000000..b125a07 --- /dev/null +++ b/docs/reference/storage/imap-filters.md @@ -0,0 +1,70 @@ +# IMAP filters + +Most storage backends support application of custom code late in delivery +process. As opposed to using SMTP pipeline modifiers or checks, it allows +modifying IMAP-specific message attributes. In particular, it allows +code to change target folder and add IMAP flags (keywords) to the message. + +There is no way to reject message using IMAP filters, this should be done +earlier in SMTP pipeline logic. Quarantined messages are not processed +by IMAP filters and are unconditionally delivered to Junk folder (or other +folder with \Junk special-use attribute). + +To use an IMAP filter, specify it in the 'imap\_filter' directive for the +used storage backend, like this: +``` +storage.imapsql local_mailboxes { + ... + + imap_filter { + command /etc/maddy/sieve.sh {account_name} + } +} +``` + +## System command filter (imap.filter.command) + +This filter is similar to check.command module +and runs a system command to obtain necessary information. + +Usage: +``` +command executable_name args... { } +``` + +Same as check.command, following placeholders are supported for command +arguments: {source\_ip}, {source\_host}, {source\_rdns}, {msg\_id}, {auth\_user}, +{sender}. Note: placeholders +in command name are not processed to avoid possible command injection attacks. + +Additionally, for imap.filter.command, {account\_name} placeholder is replaced +with effective IMAP account name, {rcpt_to}, {original_rcpt_to} provide +access to the SMTP envelope recipient (before and after any rewrites), +{subject} is replaced with the Subject header, if it is present. + +Note that if you use provided systemd units on Linux, maddy executable is +sandboxed - all commands will be executed with heavily restricted filesystem +access and other privileges. Notably, /tmp is isolated and all directories +except for /var/lib/maddy and /run/maddy are read-only. You will need to modify +systemd unit if your command needs more privileges. + +Command output should consist of zero or more lines. First one, if non-empty, overrides +destination folder. All other lines contain additional IMAP flags to add +to the message. If command wants to add flags without changing folder - first +line should be empty. + +It is valid for command to not write anything to stdout. In this case its +execution will have no effect on delivery. + +Output example: +``` +Junk +``` +In this case, message will be placed in the Junk folder. + +``` + +$Label1 +``` +In this case, message will be placed in inbox and will have +'$Label1' added. diff --git a/docs/reference/storage/imapsql.md b/docs/reference/storage/imapsql.md new file mode 100644 index 0000000..f1abbb3 --- /dev/null +++ b/docs/reference/storage/imapsql.md @@ -0,0 +1,208 @@ +# SQL-indexed storage + +The imapsql module implements database for IMAP index and message +metadata using SQL-based relational database. + +Message contents are stored in an "blob store" defined by msg_store +directive. By default this is a file system directory under /var/lib/maddy. + +Supported RDBMS: +- SQLite 3.25.0 +- PostgreSQL 9.6 or newer +- CockroachDB 20.1.5 or newer + +Account names are required to have the form of a email address (unless configured otherwise) +and are case-insensitive. UTF-8 names are supported with restrictions defined in the +PRECIS UsernameCaseMapped profile. + +``` +storage.imapsql { + driver sqlite3 + dsn imapsql.db + msg_store fs messages/ +} +``` + +imapsql module also can be used as a lookup table. +It returns empty string values for existing usernames. This might be useful +with `destination_in` directive e.g. to implement catch-all +addresses (this is a bad idea to do so, this is just an example): +``` +destination_in &local_mailboxes { + deliver_to &local_mailboxes +} +destination example.org { + modify { + replace_rcpt regexp ".*" "catchall@example.org" + } + deliver_to &local_mailboxes +} +``` + + +## Arguments + +Specify the driver and DSN. + +## Configuration directives + +### driver _string_ +**Required.**
+Default: not specified + +Use a specified driver to communicate with the database. Supported values: +sqlite3, postgres. + +Should be specified either via an argument or via this directive. + +--- + +### dsn _string_ +**Required.**
+Default: not specified + +Data Source Name, the driver-specific value that specifies the database to use. + +For SQLite3 this is just a file path. +For PostgreSQL: [https://godoc.org/github.com/lib/pq#hdr-Connection\_String\_Parameters](https://godoc.org/github.com/lib/pq#hdr-Connection\_String\_Parameters) + +Should be specified either via an argument or via this directive. + +--- + +### msg_store _store_ +Default: `fs messages/` + +Module to use for message bodies storage. + +See "Blob storage" section for what you can use here. + +--- + +### compression `off`
compression _algorithm_
compression _algorithm_ _level_ +Default: `off` + +Apply compression to message contents. +Supported algorithms: `lz4`, `zstd`. + +--- + +### appendlimit _size_ +Default: `32M` + +Don't allow users to add new messages larger than 'size'. + +This does not affect messages added when using module as a delivery target. +Use `max_message_size` directive in SMTP endpoint module to restrict it too. + +--- + +### debug _boolean_ +Default: global directive value + +Enable verbose logging. + +--- + +### junk_mailbox _name_ +Default: `Junk` + +The folder to put quarantined messages in. Thishis setting is not used if user +does have a folder with "Junk" special-use attribute. + +--- + +### disable_recent _boolean_ +Default: `true` + +Disable RFC 3501-conforming handling of \Recent flag. + +This significantly improves storage performance when SQLite3 or CockroackDB is +used at the cost of confusing clients that use this flag. + +--- + +### sqlite_cache_size _integer_ +Default: defined by SQLite + +SQLite page cache size. If positive - specifies amount of pages (1 page - 4 +KiB) to keep in cache. If negative - specifies approximate upper bound +of cache size in KiB. + +--- + +### sqlite_busy_timeout _integer_ +Default: `5000000` + +SQLite-specific performance tuning option. Amount of milliseconds to wait +before giving up on DB lock. + +--- + +### imap_filter { ... } +Default: not set + +Specifies IMAP filters to apply for messages delivered from SMTP pipeline. + +Ex. + +``` +imap_filter { + command /etc/maddy/sieve.sh {account_name} +} +``` + +--- + +### delivery_map _table_ +Default: `identity` + +Use specified table module to map recipient +addresses from incoming messages to mailbox names. + +Normalization algorithm specified in `delivery_normalize` is appied before +`delivery_map`. + +--- + +### delivery_normalize _name_ +Default: `precis_casefold_email` + +Normalization function to apply to email addresses before mapping them +to mailboxes. + +See `auth_normalize`. + +--- + +### auth_map _table_ +**Deprecated:** Use `storage_map` in imap config instead.
+Default: `identity` + +Use specified table module to map authentication +usernames to mailbox names. + +Normalization algorithm specified in auth_normalize is applied before +auth_map. + +--- + +### auth_normalize _name_ +**Deprecated:** Use `storage_map_normalize` in imap config instead.
+**Default**: `precis_casefold_email` + +Normalization function to apply to authentication usernames before mapping +them to mailboxes. + +Available options: + +- `precis_casefold_email` PRECIS UsernameCaseMapped profile + U-labels form for domain +- `precis_casefold` PRECIS UsernameCaseMapped profile for the entire string +- `precis_email` PRECIS UsernameCasePreserved profile + U-labels form for domain +- `precis` PRECIS UsernameCasePreserved profile for the entire string +- `casefold` Convert to lower case +- `noop` Nothing + +Note: On message delivery, recipient address is unconditionally normalized +using `precis_casefold_email` function. + diff --git a/docs/reference/table/auth.md b/docs/reference/table/auth.md new file mode 100644 index 0000000..4bfe4bd --- /dev/null +++ b/docs/reference/table/auth.md @@ -0,0 +1,6 @@ +# Authentication providers + +Most authentication providers are also usable as a table +that contains all usernames known to the module. Exceptions are auth.external and +pam as underlying interfaces do not define a way to check credentials +existence. diff --git a/docs/reference/table/chain.md b/docs/reference/table/chain.md new file mode 100644 index 0000000..1cbc24c --- /dev/null +++ b/docs/reference/table/chain.md @@ -0,0 +1,41 @@ +# Table chaining + +The table.chain module allows chaining together multiple table modules +by using value returned by a previous table as an input for the second +table. + +Example: +``` +table.chain { + step regexp "(.+)(\\+[^+"@]+)?@example.org" "$1@example.org" + step file /etc/maddy/emails +} +``` +This will strip +prefix from mailbox before looking it up +in /etc/maddy/emails list. + +## Configuration directives + +### step _table_ + +Adds a table module to the chain. If input value is not in the table +(e.g. file) - return "not exists" error. + +--- + +### optional_step _table_ + +Same as step but if input value is not in the table - it is passed to the +next step without changes. + +Example: +Something like this can be used to map emails to usernames +after translating them via aliases map: + +``` +table.chain { + optional_step file /etc/maddy/aliases + step regexp "(.+)@(.+)" "$1" +} +``` + diff --git a/docs/reference/table/email_localpart.md b/docs/reference/table/email_localpart.md new file mode 100644 index 0000000..19b90f1 --- /dev/null +++ b/docs/reference/table/email_localpart.md @@ -0,0 +1,20 @@ +# Email local part + +The module `table.email_localpart` extracts and unescapes local ("username") part +of the email address. + +E.g. + +* `test@example.org` => `test` +* `"test @ a"@example.org` => `test @ a` + +Mappings for invalid emails are not defined (will be treated as non-existing +values). + +``` +table.email_localpart { } +``` + +`table.email_localpart_optional` works the same, but returns non-email strings +as is. This can be used if you want to accept both `user@example.org` and +`user` somewhere and treat it the same. diff --git a/docs/reference/table/email_with_domain.md b/docs/reference/table/email_with_domain.md new file mode 100644 index 0000000..6a719e0 --- /dev/null +++ b/docs/reference/table/email_with_domain.md @@ -0,0 +1,37 @@ +# Email with domain + +The table module `table.email_with_domain` appends one or more +domains (allowing 1:N expansion) to the specified value. + +``` +table.email_with_domain DOMAIN DOMAIN... { } +``` + +It can be used to implement domain-level expansion for aliases if used together +with `table.chain`. Example: + +``` +modify { + replace_rcpt chain { + step email_local_part + step email_with_domain example.org example.com + } +} +``` + +This configuration will alias `anything@anydomain` to `anything@example.org` +and `anything@example.com`. + +It is also useful with `authorize_sender` to authorize sending using multiple +addresses under different domains if non-email usernames are used for +authentication: + +``` +check.authorize_sender { + ... + user_to_email email_with_domain example.org example.com +} +``` + +This way, user authenticated as `user` will be allowed to use +`user@example.org` or `user@example.com` as a sender address. diff --git a/docs/reference/table/file.md b/docs/reference/table/file.md new file mode 100644 index 0000000..03254f0 --- /dev/null +++ b/docs/reference/table/file.md @@ -0,0 +1,58 @@ +# File + +table.file module builds string-string mapping from a text file. + +File is reloaded every 15 seconds if there are any changes (detected using +modification time). No changes are applied if file contains syntax errors. + +Definition: +``` +file +``` +or +``` +file { + file +} +``` + +Usage example: +``` +# Resolve SMTP address aliases using text file mapping. +modify { + replace_rcpt file /etc/maddy/aliases +} +``` + +## Syntax + +Better demonstrated by examples: + +``` +# Lines starting with # are ignored. + +# And so are lines only with whitespace. + +# Whenever 'aaa' is looked up, return 'bbb' +aaa: bbb + + # Trailing and leading whitespace is ignored. + ccc: ddd + +# If there is no colon, the string is translated into "" +# That is, the following line is equivalent to +# aaa: +aaa + +# If the same key is used multiple times - table.file will return +# multiple values when queries. +ddd: firstvalue +ddd: secondvalue + +# Alternatively, multiple values can be specified +# using a comma. There is no support for escaping +# so you would have to use a different format if you require +# comma-separated values. +ddd: firstvalue, secondvalue +``` + diff --git a/docs/reference/table/regexp.md b/docs/reference/table/regexp.md new file mode 100644 index 0000000..39e873d --- /dev/null +++ b/docs/reference/table/regexp.md @@ -0,0 +1,63 @@ +# Regexp rewrite table + +The 'regexp' module implements table lookups by applying a regular expression +to the key value. If it matches - 'replacement' value is returned with $N +placeholders being replaced with corresponding capture groups from the match. +Otherwise, no value is returned. + +The regular expression syntax is the subset of PCRE. See +[https://golang.org/pkg/regexp/syntax](https://golang.org/pkg/regexp/syntax)/ for details. + +``` +table.regexp [replacement] { + full_match yes + case_insensitive yes + expand_placeholders yes +} +``` + +Note that [replacement] is optional. If it is not included - table.regexp +will return the original string, therefore acting as a regexp match check. +This can be useful in combination in `destination_in` for +advanced matching: + +``` +destination_in regexp ".*-bounce+.*@example.com" { + ... +} +``` + +## Configuration directives + +### full_match _boolean_ +Default: `yes` + +Whether to implicitly add start/end anchors to the regular expression. +That is, if `full_match` is `yes`, then the provided regular expression should +match the whole string. With `no` - partial match is enough. + +--- + +### case_insensitive _boolean_ +Default: `yes` + +Whether to make matching case-insensitive. + +--- + +### expand_placeholders _boolean_ +Default: `yes` + +Replace '$name' and '${name}' in the replacement string with contents of +corresponding capture groups from the match. + +To insert a literal $ in the output, use $$ in the template. + +## Identity table (table.identity) + +The module 'identity' is a table module that just returns the key looked up. + +``` +table.identity { } +``` + diff --git a/docs/reference/table/sql_query.md b/docs/reference/table/sql_query.md new file mode 100644 index 0000000..9b3b9eb --- /dev/null +++ b/docs/reference/table/sql_query.md @@ -0,0 +1,120 @@ +# SQL query mapping + +The table.sql_query module implements table interface using SQL queries. + +Definition: + +``` +table.sql_query { + driver + dsn + lookup + + # Optional: + init + list + add + del + set +} +``` + +Usage example: + +``` +# Resolve SMTP address aliases using PostgreSQL DB. +modify { + replace_rcpt sql_query { + driver postgres + dsn "dbname=maddy user=maddy" + lookup "SELECT alias FROM aliases WHERE address = $1" + } +} +``` + +## Configuration directives + +### driver _driver name_ +**Required.** + +Driver to use to access the database. + +Supported drivers: `postgres`, `sqlite3` (if compiled with C support) + +--- + +### dsn _data source name_ +**Required.** + +Data Source Name to pass to the driver. For SQLite3 this is just a path to DB +file. For Postgres, see +[https://pkg.go.dev/github.com/lib/pq?tab=doc#hdr-Connection\_String\_Parameters](https://pkg.go.dev/github.com/lib/pq?tab=doc#hdr-Connection\_String\_Parameters) + +--- + +### lookup _query_ +**Required.** + +SQL query to use to obtain the lookup result. + +It will get one named argument containing the lookup key. Use :key +placeholder to access it in SQL. The result row set should contain one row, one +column with the string that will be used as a lookup result. If there are more +rows, they will be ignored. If there are more columns, lookup will fail. If +there are no rows, lookup returns "no results". If there are any error - lookup +will fail. + +--- + +### init _queries..._ +Default: empty + +List of queries to execute on initialization. Can be used to configure RDBMS. + +Example, to improve SQLite3 performance: + +``` +table.sql_query { + driver sqlite3 + dsn whatever.db + init "PRAGMA journal_mode=WAL" \ + "PRAGMA synchronous=NORMAL" + lookup "SELECT alias FROM aliases WHERE address = $1" +} +``` + +--- + +### named_args _boolean_ +Default: `yes` + +Whether to use named parameters binding when executing SQL queries +or not. + +Note that maddy's PostgreSQL driver does not support named parameters and +SQLite3 driver has issues handling numbered parameters: +[https://github.com/mattn/go-sqlite3/issues/472](https://github.com/mattn/go-sqlite3/issues/472) + +--- + +### add _query_
list _query_
set _query_
del _query_ +Default: none + +If queries are set to implement corresponding table operations - table becomes +"mutable" and can be used in contexts that require writable key-value store. + +'add' query gets :key, :value named arguments - key and value strings to store. +They should be added to the store. The query **should** not add multiple values +for the same key and **should** fail if the key already exists. + +'list' query gets no arguments and should return a column with all keys in +the store. + +'set' query gets :key, :value named arguments - key and value and should replace the existing +entry in the database. + +'del' query gets :key argument - key and should remove it from the database. + +If `named_args` is set to `no` - key is passed as the first numbered parameter +($1), value is passed as the second numbered parameter ($2). + diff --git a/docs/reference/table/static.md b/docs/reference/table/static.md new file mode 100644 index 0000000..e71b448 --- /dev/null +++ b/docs/reference/table/static.md @@ -0,0 +1,21 @@ +# Static table + +The 'static' module implements table lookups using key-value pairs in its +configuration. + +``` +table.static { + entry KEY1 VALUE1 + entry KEY2 VALUE2 + ... +} +``` + +## Configuration directives + +### entry _key_ _value_ + +Add an entry to the table. + +If the same key is used multiple times, the last one takes effect. + diff --git a/docs/reference/targets/queue.md b/docs/reference/targets/queue.md new file mode 100644 index 0000000..373ff1a --- /dev/null +++ b/docs/reference/targets/queue.md @@ -0,0 +1,95 @@ +# Local queue + +Queue module buffers messages on disk and retries delivery multiple times to +another target to ensure reliable delivery. + +It is also responsible for generation of DSN messages +in case of delivery failures. + +## Arguments + +First argument specifies directory to use for storage. +Relative paths are relative to the StateDirectory. + +## Configuration directives + +``` +target.queue { + target remote + location ... + max_parallelism 16 + max_tries 4 + bounce { + destination example.org { + deliver_to &local_mailboxes + } + default_destination { + reject + } + } + + autogenerated_msg_domain example.org + debug no +} +``` + +### target _block_name_ +**Required.**
+Default: not specified + +Delivery target to use for final delivery. + +--- + +### location _directory_ +Default: `StateDirectory/configuration_block_name` + +File system directory to use to store queued messages. +Relative paths are relative to the StateDirectory. + +--- + +### max_parallelism _integer_ +Default: `16` + +Start up to _integer_ goroutines for message processing. Basically, this option +limits amount of messages tried to be delivered concurrently. + +--- + +### max_tries _integer_ +Default: `20` + +Attempt delivery up to _integer_ times. Note that no more attempts will be done +is permanent error occurred during previous attempt. + +Delay before the next attempt will be increased exponentially using the +following formula: 15mins * 1.2 ^ (n - 1) where n is the attempt number. +This gives you approximately the following sequence of delays: +18mins, 21mins, 25mins, 31mins, 37mins, 44mins, 53mins, 64mins, ... + +--- + +### bounce { ... } +Default: not specified + +This configuration contains pipeline configuration to be used for generated DSN +(Delivery Status Notification) messages. + +If this is block is not present in configuration, DSNs will not be generated. +Note, however, this is not what you want most of the time. + +--- + +### autogenerated_msg_domain _domain_ +Default: global directive value + +Domain to use in sender address for DSNs. Should be specified too if 'bounce' +block is specified. + +--- + +### debug _boolean_ +Default: `no` + +Enable verbose logging. \ No newline at end of file diff --git a/docs/reference/targets/remote.md b/docs/reference/targets/remote.md new file mode 100644 index 0000000..9a1b606 --- /dev/null +++ b/docs/reference/targets/remote.md @@ -0,0 +1,295 @@ +# Remote MX delivery + +Module that implements message delivery to remote MTAs discovered via DNS MX +records. You probably want to use it with queue module for reliability. + +If a message check marks a message as 'quarantined', remote module +will refuse to deliver it. + +## Configuration directives + +``` +target.remote { + hostname mx.example.org + debug no +} +``` + +### hostname _domain_ +Default: global directive value + +Hostname to use client greeting (EHLO/HELO command). Some servers require it to +be FQDN, SPF-capable servers check whether it corresponds to the server IP +address, so it is better to set it to a domain that resolves to the server IP. + +--- + +### limits { ... } +Default: no limits + +See ['limits' directive for SMTP endpoint](/reference/endpoints/smtp/#rate-concurrency-limiting). +It works the same except for address domains used for +per-source/per-destination are as observed when message exits the server. + +--- + +### local_ip _ip-address_ +Default: empty + +Choose the local IP to bind for outbound SMTP connections. + +--- + +### force_ipv4 _boolean_ +Default: `false` + +Force resolving outbound SMTP domains to IPv4 addresses. Some server providers +do not offer a way to properly set reverse PTR domains for IPv6 addresses; this +option makes maddy only connect to IPv4 addresses so that its public IPv4 address +is used to connect to that server, and thus reverse PTR checks are made against +its IPv4 address. + +Warning: this may break sending outgoing mail to IPv6-only SMTP servers. + +--- + +### connect_timeout _duration_ +Default: `5m` + +Timeout for TCP connection establishment. + +RFC 5321 recommends 5 minutes for "initial greeting" that includes TCP +handshake. maddy uses two separate timers - one for "dialing" (DNS A/AAAA +lookup + TCP handshake) and another for "initial greeting". This directive +configures the former. The latter is not configurable and is hardcoded to be +5 minutes. + +--- + +### command_timeout _duration_ +Default: `5m` + +Timeout for any SMTP command (EHLO, MAIL, RCPT, DATA, etc). + +If STARTTLS is used this timeout also applies to TLS handshake. + +RFC 5321 recommends 5 minutes for MAIL/RCPT and 3 minutes for +DATA. + +--- + +### submission_timeout _duration_ +Default: `12m` + +Time to wait after the entire message is sent (after "final dot"). + +RFC 5321 recommends 10 minutes. + +--- + +### debug _boolean_ +Default: global directive value + +Enable verbose logging. + +--- + +### requiretls_override _boolean_ +Default: `true` + +Allow local security policy to be disabled using 'TLS-Required' header field in +sent messages. Note that the field has no effect if transparent forwarding is +used, message body should be processed before outbound delivery starts for it +to take effect (e.g. message should be queued using 'queue' module). + +--- + +### relaxed_requiretls _boolean_ +Default: `true` + +This option disables strict conformance with REQUIRETLS specification and +allows forwarding of messages 'tagged' with REQUIRETLS to MXes that are not +advertising REQUIRETLS support. It is meant to allow REQUIRETLS use without the +need to have support from all servers. It is based on the assumption that +server referenced by MX record is likely the final destination and therefore +there is only need to secure communication towards it and not beyond. + +--- + +### conn_reuse_limit _integer_ +Default: `10` + +Amount of times the same SMTP connection can be used. +Connections are never reused if the previous DATA command failed. + +--- + +### conn_max_idle_count _integer_ +Default: `10` + +Max. amount of idle connections per recipient domains to keep in cache. + +--- + +### conn_max_idle_time _integer_ +Default: `150` (2.5 min) + +Amount of time the idle connection is still considered potentially usable. + +--- + +## Security policies + +### mx_auth { ... } +Default: no policies + +'remote' module implements a number of of schemes and protocols necessary to +ensure security of message delivery. Most of these schemes are concerned with +authentication of recipient server and TLS enforcement. + +To enable mechanism, specify its name in the `mx_auth` directive block: + +``` +mx_auth { + dane + mtasts +} +``` + +Additional configuration is possible if supported by the mechanism by +specifying additional options as a block for the corresponding mechanism. +E.g. + +``` +mtasts { + cache ram +} +``` + +If the `mx_auth` directive is not specified, no mechanisms are enabled. Note +that, however, this makes outbound SMTP vulnerable to a numerous downgrade +attacks and hence not recommended. + +It is possible to share the same set of policies for multiple 'remote' module +instances by defining it at the top-level using `mx_auth` module and then +referencing it using standard & syntax: + +``` +mx_auth outbound_policy { + dane + mtasts { + cache ram + } +} + +# ... somewhere else ... + +deliver_to remote { + mx_auth &outbound_policy +} + +# ... somewhere else ... + +deliver_to remote { + mx_auth &outbound_policy + tls_client { ... } +} +``` + +--- + +### MTA-STS + +Checks MTA-STS policy of the recipient domain. Provides proper authentication +and TLS enforcement for delivery, but partially vulnerable to persistent active +attacks. + +Sets MX level to "mtasts" if the used MX matches MTA-STS policy even if it is +not set to "enforce" mode. + +``` +mtasts { + cache fs + fs_dir StateDirectory/mtasts_cache +} +``` + +### cache `fs` | `ram` +Default: `fs` + +Storage to use for MTA-STS cache. 'fs' is to use a filesystem directory, 'ram' +to store the cache in memory. + +It is recommended to use 'fs' since that will not discard the cache (and thus +cause MTA-STS security to disappear) on server restart. However, using the RAM +cache can make sense for high-load configurations with good uptime. + +### fs_dir _directory_ +Default: `StateDirectory/mtasts_cache` + +Filesystem directory to use for policies caching if 'cache' is set to 'fs'. + +--- + +### DNSSEC + +Checks whether MX records are signed. Sets MX level to "dnssec" is they are. + +maddy does not validate DNSSEC signatures on its own. Instead it relies on +the upstream resolver to do so by causing lookup to fail when verification +fails and setting the AD flag for signed and verified zones. As a safety +measure, if the resolver is not 127.0.0.1 or ::1, the AD flag is ignored. + +DNSSEC is currently not supported on Windows and other platforms that do not +have the /etc/resolv.conf file in the standard format. + +``` +dnssec { } +``` + +--- + +### DANE + +Checks TLSA records for the recipient MX. Provides downgrade-resistant TLS +enforcement. + +Sets TLS level to "authenticated" if a valid and matching TLSA record uses +DANE-EE or DANE-TA usage type. + +See above for notes on DNSSEC. DNSSEC support is required for DANE to work. + +``` +dane { } +``` + +--- + +### Local policy + +Checks effective TLS and MX levels (as set by other policies) against local +configuration. + +``` +local_policy { + min_tls_level none + min_mx_level none +} +``` + +Using `local_policy off` is equivalent to setting both directives to `none`. + +### min_tls_level `none` | `encrypted` | `authenticated` +Default: `encrypted` + +Set the minimal TLS security level required for all outbound messages. + +See [Security levels](/seclevels) page for details. + +### min_mx_level `none` | `mtasts` | `dnssec` +Default: `none` + +Set the minimal MX security level required for all outbound messages. + +See [Security levels](/seclevels) page for details. + diff --git a/docs/reference/targets/smtp.md b/docs/reference/targets/smtp.md new file mode 100644 index 0000000..ebbc430 --- /dev/null +++ b/docs/reference/targets/smtp.md @@ -0,0 +1,123 @@ +# SMTP & LMTP transparent forwarding + +Module that implements transparent forwarding of messages over SMTP. + +Use in pipeline configuration: + +``` +deliver_to smtp tcp://127.0.0.1:5353 +# or +deliver_to smtp tcp://127.0.0.1:5353 { + # Other settings, see below. +} +``` + +target.lmtp can be used instead of target.smtp to +use LMTP protocol. + +Endpoint addresses use format described in [Configuration files syntax / Address definitions](/reference/config-syntax/#address-definitions). + +## Configuration directives + +``` +target.smtp { + debug no + tls_client { + ... + } + attempt_starttls yes + require_tls no + auth off + targets tcp://127.0.0.1:2525 + connect_timeout 5m + command_timeout 5m + submission_timeout 12m +} +``` + +### debug _boolean_ +Default: global directive value + +Enable verbose logging. + +--- + +### tls_client { ... } +Default: not specified + +Advanced TLS client configuration options. See [TLS configuration / Client](/reference/tls/#client) for details. + +--- + +### starttls _boolean_ +Default: `yes` (`no` for `target.lmtp`) + +Use STARTTLS to enable TLS encryption. If STARTTLS is not supported +by the remote server - connection will fail. + +maddy will use `localhost` as HELO hostname before STARTTLS +and will only send its actual hostname after STARTTLS. + +### attempt_starttls _boolean_ +Default: `yes` (`no` for `target.lmtp`) + +DEPRECATED: Equivalent to `starttls`. Plaintext fallback is no longer +supported. + +--- + +### require_tls _boolean_ +Default: `no` + +DEPRECATED: Ignored. Set `starttls yes` to use STARTLS. + +--- + +### auth `off` | `plain` _username_ _password_ | `forward` | `external` +Default: `off` + +Specify the way to authenticate to the remote server. +Valid values: + +- `off` – No authentication. +- `plain` – Authenticate using specified username-password pair. + **Don't use** this without enforced TLS (`require_tls`). +- `forward` – Forward credentials specified by the client. + **Don't use** this without enforced TLS (`require_tls`). +- `external` – Request "external" SASL authentication. This is usually used for + authentication using TLS client certificates. See [TLS configuration / Client](/reference/tls/#client) for details. + +--- + +### targets _endpoints..._ +**Required.**
+Default: not specified + +List of remote server addresses to use. See [Address definitions](/reference/config-syntax/#address-definitions) +for syntax to use. Basically, it is `tcp://ADDRESS:PORT` +for plain SMTP and `tls://ADDRESS:PORT` for SMTPS (aka SMTP with Implicit +TLS). + +Multiple addresses can be specified, they will be tried in order until connection to +one succeeds (including TLS handshake if TLS is required). + +--- + +### connect_timeout _duration_ +Default: `5m` + +Same as for target.remote. + +--- + +### command_timeout _duration_ +Default: `5m` + +Same as for target.remote. + +--- + +### submission_timeout _duration_ +Default: `12m` + +Same as for target.remote. diff --git a/docs/reference/tls-acme.md b/docs/reference/tls-acme.md new file mode 100644 index 0000000..a7be47d --- /dev/null +++ b/docs/reference/tls-acme.md @@ -0,0 +1,290 @@ +# Automatic certificate management via ACME + +Maddy supports obtaining certificates using ACME protocol. + +To use it, create a configuration name for `tls.loader.acme` +and reference it from endpoints that should use automatically +configured certificates: + +``` +tls.loader.acme local_tls { + email put-your-email-here@example.org + agreed # indicate your agreement with Let's Encrypt ToS + challenge dns-01 +} + +smtp tcp://127.0.0.1:25 { + tls &local_tls + ... +} +``` + +You can also use a global `tls` directive to use automatically +obtained certificates for all endpoints: + +``` +tls { + loader acme { + email maddy-acme@example.org + agreed + challenge dns-01 + } +} +``` + +Note: `tls &local_tls` as a global directive won't work because +global directives are initialized before other configuration blocks. + +Currently the only supported challenge is `dns-01` one therefore +you also need to configure the DNS provider: + +``` +tls.loader.acme local_tls { + email maddy-acme@example.org + agreed + challenge dns-01 + dns PROVIDER_NAME { + ... + } +} +``` + +See below for supported providers and necessary configuration +for each. + +## Configuration directives + +``` +tls.loader.acme { + debug off + hostname example.maddy.invalid + store_path /var/lib/maddy/acme + ca https://acme-v02.api.letsencrypt.org/directory + test_ca https://acme-staging-v02.api.letsencrypt.org/directory + email test@maddy.invalid + agreed off + challenge dns-01 + dns ... +} +``` + +### debug _boolean_ +Default: global directive value + +Enable debug logging. + +--- + +### hostname _str_ +**Required.**
+Default: global directive value + +Domain name to issue certificate for. + +--- + +### store_path _path_ +Default: `state_dir/acme` + +Where to store issued certificates and associated metadata. +Currently only filesystem-based store is supported. + +--- + +### ca _url_ +Default: Let's Encrypt production CA + +URL of ACME directory to use. + +--- + +### test_ca _url_ +Default: Let's Encrypt staging CA + +URL of ACME directory to use for retries should +primary CA fail. + +maddy will keep attempting to issues certificates +using `test_ca` until it succeeds then it will switch +back to the one configured via 'ca' option. + +This avoids rate limit issues with production CA. + +--- + +### override_domain _domain_ +Default: not set + +Override the domain to set the TXT record on for DNS-01 challenge. +This is to delegate the challenge to a different domain. + +See https://www.eff.org/deeplinks/2018/02/technical-deep-dive-securing-automation-acme-dns-challenge-validation +for explanation why this might be useful. + +--- + +### email _str_ +Default: not set + +Email to pass while registering an ACME account. + +--- + +### agreed _boolean_ +Default: false + +Whether you agreed to ToS of the CA service you are using. + +--- + +### challenge `dns-01` +Default: not set + +Challenge(s) to use while performing domain verification. + +## DNS providers + +Support for some providers is not provided by standard builds. +To be able to use these, you need to compile maddy +with "libdns_PROVIDER" build tag. +E.g. +``` +./build.sh --tags 'libdns_googleclouddns' +``` + +- gandi + +``` +dns gandi { + api_token "token" +} +``` + +- digitalocean + +``` +dns digitalocean { + api_token "..." +} +``` + +- cloudflare + +See [https://github.com/libdns/cloudflare#authenticating](https://github.com/libdns/cloudflare#authenticating) + +``` +dns cloudflare { + api_token "..." +} +``` + +- vultr + +``` +dns vultr { + api_token "..." +} +``` + +- hetzner + +``` +dns hetzner { + api_token "..." +} +``` + +- namecheap + +``` +dns namecheap { + api_key "..." + api_username "..." + + # optional: API endpoint, production one is used if not set. + endpoint "https://api.namecheap.com/xml.response" + + # optional: your public IP, discovered using icanhazip.com if not set + client_ip 1.2.3.4 +} +``` + +- googleclouddns (non-default) + +``` +dns googleclouddns { + project "project_id" + service_account_json "path" +} +``` + +- route53 (non-default) + +``` +dns route53 { + secret_access_key "..." + access_key_id "..." + # or use environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +} +``` + +- leaseweb (non-default) + +``` +dns leaseweb { + api_key "key" +} +``` + +- metaname (non-default) + +``` +dns metaname { + api_key "key" + account_ref "reference" +} +``` + +- alidns (non-default) + +``` +dns alidns { + key_id "..." + key_secret "..." +} +``` + +- namedotcom (non-default) + +``` +dns namedotcom { + user "..." + token "..." +} +``` + +- rfc2136 (non-default) + +``` +dns rfc2136 { + key_name "..." + # Secret + key "..." + # HMAC algorithm used to generate the key, lowercase, e.g. hmac-sha512 + key_alg "..." + # server to which the dynamic update will be sent, e.g. 127.0.0.1 + # you can also specify the port: 127.0.0.1:53 + server "..." +} +``` + +- acmedns (non-default) + +``` +dns acmedns { + username "..." + password "..." + subdomain "..." + server_url "..." +} +``` diff --git a/docs/reference/tls.md b/docs/reference/tls.md new file mode 100644 index 0000000..954b0e0 --- /dev/null +++ b/docs/reference/tls.md @@ -0,0 +1,155 @@ +# TLS configuration + +## Server-side + +TLS certificates are obtained by modules called "certificate loaders". 'tls' directive +arguments specify name of loader to use and arguments. Due to syntax limitations +advanced configuration for loader should be specified using 'loader' directive, see +below. + +``` +tls file cert.pem key.pem { + protocols tls1.2 tls1.3 + curves X25519 + ciphers ... +} + +tls { + loader file cert.pem key.pem { + # Options for loader go here. + } + protocols tls1.2 tls1.3 + curves X25519 + ciphers ... +} +``` + +### Available certificate loaders + +- `file` – Accepts argument pairs specifying certificate and then key. + E.g. `tls file certA.pem keyA.pem certB.pem keyB.pem`. + If multiple certificates are listed, SNI will be used. +- `acme` – Automatically obtains a certificate using ACME protocol (Let's Encrypt) +- `off` – Not really a loader but a special value for tls directive, + explicitly disables TLS for endpoint(s). + +## Advanced TLS configuration + +**Note: maddy uses secure defaults and TLS handshake is resistant to active downgrade attacks. There is no need to change anything in most cases.** + +--- + +### protocols _min-version_ _max-version_ | _version_ +Default: `tls1.0 tls1.3` + +Minimum/maximum accepted TLS version. If only one value is specified, it will +be the only one usable version. + +Valid values are: `tls1.0`, `tls1.1`, `tls1.2`, `tls1.3` + +--- + +### ciphers _ciphers..._ +Default: Go version-defined set of 'secure ciphers', ordered by hardware +performance + +List of supported cipher suites, in preference order. Not used with TLS 1.3. + +Valid values: + +- `RSA-WITH-RC4128-SHA` +- `RSA-WITH-3DES-EDE-CBC-SHA` +- `RSA-WITH-AES128-CBC-SHA` +- `RSA-WITH-AES256-CBC-SHA` +- `RSA-WITH-AES128-CBC-SHA256` +- `RSA-WITH-AES128-GCM-SHA256` +- `RSA-WITH-AES256-GCM-SHA384` +- `ECDHE-ECDSA-WITH-RC4128-SHA` +- `ECDHE-ECDSA-WITH-AES128-CBC-SHA` +- `ECDHE-ECDSA-WITH-AES256-CBC-SHA` +- `ECDHE-RSA-WITH-RC4128-SHA` +- `ECDHE-RSA-WITH-3DES-EDE-CBC-SHA` +- `ECDHE-RSA-WITH-AES128-CBC-SHA` +- `ECDHE-RSA-WITH-AES256-CBC-SHA` +- `ECDHE-ECDSA-WITH-AES128-CBC-SHA256` +- `ECDHE-RSA-WITH-AES128-CBC-SHA256` +- `ECDHE-RSA-WITH-AES128-GCM-SHA256` +- `ECDHE-ECDSA-WITH-AES128-GCM-SHA256` +- `ECDHE-RSA-WITH-AES256-GCM-SHA384` +- `ECDHE-ECDSA-WITH-AES256-GCM-SHA384` +- `ECDHE-RSA-WITH-CHACHA20-POLY1305` +- `ECDHE-ECDSA-WITH-CHACHA20-POLY1305` + +--- + +### curves _curves..._ +Default: defined by Go version + +The elliptic curves that will be used in an ECDHE handshake, in preference +order. + +Valid values: `p256`, `p384`, `p521`, `X25519`. + +## Client + +`tls_client` directive allows to customize behavior of TLS client implementation, +notably adjusting minimal and maximal TLS versions and allowed cipher suites, +enabling TLS client authentication. + +``` +tls_client { + protocols tls1.2 tls1.3 + ciphers ... + curves X25519 + root_ca /etc/ssl/cert.pem + + cert /etc/ssl/private/maddy-client.pem + key /etc/ssl/private/maddy-client.pem +} +``` + +--- + +### protocols _min-version_ _max-version_ | _version_ +Default: `tls1.0 tls1.3` + +Minimum/maximum accepted TLS version. If only one value is specified, it will +be the only one usable version. + +Valid values are: `tls1.0`, `tls1.1`, `tls1.2`, `tls1.3` + +--- + +### ciphers _ciphers..._ +Default: Go version-defined set of 'secure ciphers', ordered by hardware +performance + +List of supported cipher suites, in preference order. Not used with TLS 1.3. + +See TLS server configuration for list of supported values. + +--- + +### curves _curves..._ +Default: defined by Go version + +The elliptic curves that will be used in an ECDHE handshake, in preference +order. + +Valid values: `p256`, `p384`, `p521`, `X25519`. + +--- + +### root_ca _paths..._ +Default: system CA pool + +List of files with PEM-encoded CA certificates to use when verifying +server certificates. + +--- + +### cert _cert-path_
key _key-path_ +Default: not specified + +Present the specified certificate when server requests a client certificate. +Files should use PEM format. Both directives should be specified. diff --git a/docs/seclevels.md b/docs/seclevels.md new file mode 100644 index 0000000..984be4a --- /dev/null +++ b/docs/seclevels.md @@ -0,0 +1,89 @@ +# Outbound delivery security + +maddy implements a number of schemes and protocols for discovery and +enforcement of security features supported by the recipient MTA. + +## Introduction to the problems of secure SMTP + +Outbound delivery security involves two independent problems: + +- MX record authentication +- TLS enforcement + +### MX record authentication + +When MTA wants to deliver a message to a mailbox at remote domain, it needs to +discover the server to use for it. It is done through the lookup of DNS MX +records for the recipient. + +Problem arises from the fact that DNS does not have any cryptographic +protection and so any malicious actor can technically modify the response to +contain any server. And MTA would use that server! + +There are two protocols that solve this problem: MTA-STS and DNSSEC. +Former requires the MTA to verify used records against a list of rules published +via HTTPS. Later cryptographically signs the records themselves. + +### TLS enforcement + +By default, server-server SMTP is unencrypted. If remote server supports TLS, +it is advertised via the ESMTP extension named STARTTLS, but malicious actor +controlling communication channel can hide the support for STARTTLS and sender +MTA will have to use plaintext. There needs to be a out-of-band authenticated +channel to indicate TLS support (and to require its use). + +MTA-STS and DANE solve this problem. In the first case, if policy is in +"enforce" mode then MTA is required to use TLS when delivering messages to a +remote server. DANE does pretty much the same thing, but using DNSSEC-signed +TLSA records. + +## maddy policy details + +maddy defines two values indicating how "secure" delivery of message will be: + +- MX security level +- TLS security level + +These values correspond to the problems described above. On delivery, the +established connection to the remote server is "ranked" using these values and +then they are compared against a number of policies (including local +configuration). If the effective value is lower than the required one, the +connection is closed and next candidate server is used. If all connections fail +this way - the delivery is failed (or deferred if there was a temporary error +when checking policies). + +Below is the table summarizing the security level values defined in maddy and +protection they offer. + +| MX/TLS level | None | Encrypted | Authenticated | +| ------------- | ---- | --------- | -------------------- | +| None | - | P | P | +| MTA-STS | - | P | PA (see note 1) | +| DNSSEC | - | P | PA | + +Legend: P - protects against passive attacks; A - protects against active +attacks + +- MX level: None. MX candidate was returned as a result of DNS lookup for the + recipient domain, no additional checks done. +- MX level: MTA-STS. Used MX matches the MTA-STS policy published by the + recipient domain (even one in testing mode). +- MX level: DNSSEC. MX record is signed. + +- TLS level: None. Plaintext connection was established, TLS is not available + or failed. +- TLS level: Encrypted. TLS connection was established, the server certificate + failed X.509 and DANE verification. +- TLS level: Authenticated. TLS connection was established, the server + certificate passes X.509 **or** DANE verification. + +**Note 1:** Persistent attacker able to control network connection can +interfere with policy refresh, downgrading protection to be secure only against +passive attacks. + +## maddy security policies + +See [Remote MX delivery](/reference/targets/remote/) for description of configuration options available for each policy mechanism +supported by maddy. + +[RFC 8461 Section 10.2]: https://www.rfc-editor.org/rfc/rfc8461.html#section-10.2 (SMTP MTA Strict Transport Security - 10.2. Preventing Policy Discovery) diff --git a/docs/third-party/dovecot.md b/docs/third-party/dovecot.md new file mode 100644 index 0000000..22d51c3 --- /dev/null +++ b/docs/third-party/dovecot.md @@ -0,0 +1,87 @@ +# Dovecot + +Builtin maddy IMAP server may not match your requirements in terms of +performance, reliability or anything. For this reason it is possible to +integrate it with any external IMAP server that implements necessary +protocols. Here is how to do it for Dovecot. + +1. Get rid of `imap` endpoint and existing `local_authdb` and `local_mailboxes` + blocks. + +2. Setup Dovecot to provide LMTP endpoint + +Here is an example configuration snippet: +``` +# /etc/dovecot/dovecot.conf +protocols = imap lmtp + +# /etc/dovecot/conf.d/10-master.conf +service lmtp { + unix_listener lmtp-maddy { + mode = 0600 + user = maddy + } +} +``` + +Add `local_mailboxes` block to maddy config using `target.lmtp` module: +``` +target.lmtp local_mailboxes { + targets unix:///var/run/dovecot/lmtp-maddy +} +``` + +### Authentication + +In addition to MTA service, maddy also provides Submission service, but it +needs authentication provider data to work correctly, maddy can use Dovecot +SASL authentication protocol for it. + +You need the following in Dovecot's `10-master.conf`: +``` +service auth { + unix_listener auth-maddy-client { + mode = 0660 + user = maddy + } +} +``` + +Then just configure `dovecot_sasl` module for `submission`: +``` +submission ... { + auth dovecot_sasl unix:///var/run/dovecot/auth-maddy-client + ... other configuration ... +} +``` + +## Other IMAP servers + +Integration with other IMAP servers might be more problematic because there is +no standard protocol for authentication delegation. You might need to configure +the IMAP server to implement MSA functionality by forwarding messages to maddy +for outbound delivery. This might require more configuration changes on maddy +side since by default it will not allow relay on port 25 even for localhost +addresses. The easiest way is to create another SMTP endpoint on some port +(probably Submission port): +``` +smtp tcp://127.0.0.1:587 { + deliver_to &remote_queue +} +``` +And configure IMAP server's Submission service to forward outbound messages +there. + +Depending on how Submission service is implemented you may also need to route +messages for local domains back to it via LMTP: +``` +smtp tcp://127.0.0.1:587 { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + deliver_to &remote_queue + } +} +``` + diff --git a/docs/third-party/mailman3.md b/docs/third-party/mailman3.md new file mode 100644 index 0000000..a29d71e --- /dev/null +++ b/docs/third-party/mailman3.md @@ -0,0 +1,84 @@ +# Mailman 3 + +Setting up Mailman 3 with maddy involves some additional work as compared to +other MTAs as there is no Python package in Mailman suite that can generate +address lists in format supported by maddy. + +We assume you are already familiar with Mailman configuration guidelines and +how stuff works in general/for other MTAs. + +## Accepting messages + +First of all, you need to use NullMTA package for mta.incoming so Mailman will +not try to generate any configs. LMTP listener is configured as usual. +``` +[mta] +incoming: mailman.mta.null.NullMTA +lmtp_host: 127.0.0.1 +lmtp_port: 8024 +``` + +After that, you will need to configure maddy to send messages to Mailman. + +The preferable way of doing so is destination_in and table.regexp: +``` +msgpipeline local_routing { + destination_in regexp "first-mailinglist(-(bounces\+.*|confirm\+.*|join|leave|owner|request|subscribe|unsubscribe))?@lists.example.org" { + deliver_to lmtp tcp://127.0.0.1:8024 + } + destination_in regexp "second-mailinglist(-(bounces\+.*|confirm\+.*|join|leave|owner|request|subscribe|unsubscribe))?@lists.example.org" { + deliver_to lmtp tcp://127.0.0.1:8024 + } + + ... +} +``` + +A more simple option is also meaningful (provided you have a separate domain +for lists): +``` +msgpipeline local_routing { + destination lists.example.org { + deliver_to lmtp tcp://127.0.0.1:8024 + } + + ... +} +``` +But this variant will lead to inefficient handling of non-existing subaddresses. +See [Mailman Core issue 14](https://gitlab.com/mailman/mailman/-/issues/14) for +details. (5 year old issue, sigh...) + +## Sending messages + +It is recommended to configure Mailman to send messages using Submission port +with authentication and TLS as maddy does not allow relay on port 25 for local +clients as some MTAs do: +``` +[mta] +# ... incoming configuration here ... +outgoing: mailman.mta.deliver.deliver +smtp_host: mx.example.org +smtp_port: 465 +smtp_user: mailman@example.org +smtp_pass: something-very-secret +smtp_secure_mode: smtps +``` + +If you do not want to use TLS and/or authentication you can create a separate +endpoint and just point Mailman to it. E.g. +``` +smtp tcp://127.0.0.1:2525 { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + deliver_to &remote_queue + } +} +``` + +Note that if you use a separate domain for lists, it need to be included in +local_domains macro in default config. This will ensure maddy signs messages +using DKIM for outbound messages. It is also highly recommended to configure +ARC in Mailman 3. diff --git a/docs/third-party/rspamd.md b/docs/third-party/rspamd.md new file mode 100644 index 0000000..3d2ce48 --- /dev/null +++ b/docs/third-party/rspamd.md @@ -0,0 +1,38 @@ +# rspamd + +maddy has direct support for rspamd HTTP protocol. There is no need to use +milter proxy. + +If rspamd is running locally, it is enough to just add `rspamd` check +with default configuration into appropriate check block (probably in +local_routing): +``` +check { + ... + rspamd +} +``` + +You might want to disable builtin SPF, DKIM and DMARC for performance +reasons but note that at the moment, maddy will not generate +Authentication-Results field with rspamd results. + +If rspamd is not running on a local machine, change api_path to point +to the "normal" worker socket: + +``` +check { + ... + rspamd { + api_path http://spam-check.example.org:11333 + } +} +``` + +Default mapping of rspamd action -> maddy action is as follows: + +- "add header" => Quarantine +- "rewrite subject" => Quarantine +- "soft reject" => Reject with temporary error +- "reject" => Reject with permanent error +- "greylist" => Ignored \ No newline at end of file diff --git a/docs/third-party/smtp-servers.md b/docs/third-party/smtp-servers.md new file mode 100644 index 0000000..599a00d --- /dev/null +++ b/docs/third-party/smtp-servers.md @@ -0,0 +1,58 @@ +# External SMTP server + +It is possible to use maddy as an IMAP server only and have it interface with +external SMTP server using standard protocols. + +Here is the minimal configuration that creates a local IMAP index, credentials +database and IMAP endpoint: +``` +# Credentials DB. +table.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } +} + +# IMAP storage/index. +storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db +} + +# IMAP endpoint using these above. +imap tls://0.0.0.0:993 tcp://0.0.0.0:143 { + auth &local_authdb + storage &local_mailboxes +} +``` + +To accept local messages from an external SMTP server +it is possible to create an LMTP endpoint: +``` +# LMTP endpoint on Unix socket delivering to IMAP storage +# in previous config snippet. +lmtp unix:/run/maddy/lmtp.sock { + hostname mx.maddy.test + + deliver_to &local_mailboxes +} +``` + +Look up documentation for your SMTP server on how to make it +send messages using LMTP to /run/maddy/lmtp.sock. + +To handle authentication for Submission (client-server SMTP) SMTP server +needs to access credentials database used by maddy. maddy implements +server side of Dovecot authentication protocol so you can use +it if SMTP server implements "Dovecot SASL" client. + +To create a Dovecot-compatible sasld endpoint, add the following configuration +block: +``` +# Dovecot-compatible sasld endpoint using data from local_authdb. +dovecot_sasld unix:/run/maddy/auth-client.sock { + auth &local_authdb +} +``` diff --git a/docs/tutorials/alias-to-remote.md b/docs/tutorials/alias-to-remote.md new file mode 100644 index 0000000..ddbc76b --- /dev/null +++ b/docs/tutorials/alias-to-remote.md @@ -0,0 +1,123 @@ +# Forward messages to a remote MX + +Default maddy configuration is done in a way that does not result in any +outbound messages being sent as a result of port 25 traffic. + +In particular, this means that if you handle messages for example.org but not +example.com and have the following in your aliases file (e.g. /etc/maddy/aliases): + +``` +foxcpp@example.org: foxcpp@example.com +``` + +You will get "User does not exist" error when attempting to send a message to +foxcpp@example.org because foxcpp@example.com does not exist on as a local +user. + +Some users may want to make it work, but it is important to understand the +consequences of such configuration: + +- Flooding your server will also flood the remote server. +- If your spam filtering is not good enough, you will send spam to the remote + server. + +In both cases, you might harm the reputation of your server (e.g. get your IP +listed in a DNSBL). + +**So, this is a bad practice. Do so only if you clearly understand the +consequences (including the Bounce handling section below).** + +If you want to do it anyway, here is the part of the configuration that needs +tweaking: + +``` +msgpipeline local_routing { + destination postmaster $(local_domains) { + modify { + replace_rcpt regexp "(.+)\+(.+)@(.+)" "$1@$3" + replace_rcpt file /etc/maddy/aliases + } + + deliver_to &local_mailboxes + } + + default_destination { + reject 550 5.1.1 "User doesn't exist" + } +} +``` + +In default configuration, `local_routing` block is responsible for handling +messages that are received via SMTP or Submission and have the initial +destination address at a local domain. + +Note the `modify { }` block being nested inside `destination` and then followed +by unconditional `deliver_to &local_mailboxes`. This means: if address is +on `$(local_domains)`, apply aliases and deliver to mailboxes from +`&local_mailboxes`. + +The problem here is that recipients are matched before aliases are resolved so +in the end, maddy attempts to look up foxcpp@example.com locally. The solution +is to insert another step into the pipeline configuration to rerun matching +*after* aliases are resolved. This can be done using the 'reroute' directive: + +``` +msgpipeline local_routing { + destination postmaster $(local_domains) { + modify { + replace_rcpt file /etc/maddy/aliases + ... + } + + reroute { + destination postmaster $(local_domains) { + deliver_to &local_mailboxes + } + default_destination { + deliver_to &remote_queue + } + } + } + + default_destination { + reject 550 5.1.1 "User doesn't exist" + } +} +``` + +## Bounce handling + +Once the message is delivered to `remote_queue`, it will follow the usual path +for outbound delivery, including queuing and multiple attempts. This also +means bounce messages will be generated on failures. When accepting messages +from arbitrary senders via the 25 port, the DSN recipient will be whatever +sender specifies in the MAIL FROM command. This is prone to [collateral spam] +when an automatically generated bounce message gets sent to a spoofed address. + +However, the default maddy configuration ensures that in this case, the NDN +will be delivered only if the original sender is a local user. Backscatter can +not happen if the sender spoofed a local address since such messages will not +be accepted in the first place. + +You can also configure maddy to send bounce messages to remote +addresses, but in this case, you should configure a really strict local policy +to make sure the sender address is not spoofed. There is no detailed +explanation of how to do this since this is a terrible idea in general. + +[collateral spam]: https://en.wikipedia.org/wiki/Backscatter_(e-mail) + +## Transparent forwarding + +As an alternative to silently dropping messages on remote delivery failures, +you might want to use transparent forwarding and reject the message without +accepting it first ("connection-stage rejection"). + +To do so, simply do not use the queue, replace +``` +deliver_to &remote_queue +``` +with +``` +deliver_to &outbound_delivery +``` +(assuming outbound_delivery refers to target.remote block) diff --git a/docs/tutorials/building-from-source.md b/docs/tutorials/building-from-source.md new file mode 100644 index 0000000..55b6852 --- /dev/null +++ b/docs/tutorials/building-from-source.md @@ -0,0 +1,55 @@ +# Building from source + +## System dependencies + +You need C toolchain, Go toolchain and Make: + +On Debian-based system this should work: +``` +apt-get install golang-1.23 gcc libc6-dev make +``` + +Additionally, if you want manual pages, you should also have scdoc installed. +Figuring out the appropriate way to get scdoc is left as an exercise for +reader (for Ubuntu 22.04 LTS it is in repositories). + +## Recent Go toolchain + +maddy depends on a rather recent Go toolchain version that may not be +available in some distributions (*cough* Debian *cough*). + +`go` command in Go 1.21 or newer will automatically download up-to-date +toolchain to build maddy. It is necessary to run commands below only +if you have `go` command version older than 1.21. + +``` +wget "https://go.dev/dl/go1.23.5.linux-amd64.tar.gz" +tar xf "go1.23.5.linux-amd64.tar.gz" +export GOROOT="$PWD/go" +export PATH="$PWD/go/bin:$PATH" +``` + +## Step-by-step + +1. Clone repository +``` +$ git clone https://github.com/foxcpp/maddy.git +$ cd maddy +``` + +2. Select the appropriate version to build: +``` +$ git checkout v0.8.0 # a specific release +$ git checkout master # next bugfix release +$ git checkout dev # next feature release +``` + +3. Build & install it +``` +$ ./build.sh +$ sudo ./build.sh install +``` + +4. Finish setup as described in [Setting up](../setting-up) (starting from System configuration). + + diff --git a/docs/tutorials/pam.md b/docs/tutorials/pam.md new file mode 100644 index 0000000..a189a33 --- /dev/null +++ b/docs/tutorials/pam.md @@ -0,0 +1,94 @@ +# Using PAM authentication + +maddy supports user authentication using PAM infrastructure via `auth.pam` +module. + +In order to use it, however, either maddy itself should be compiled +with libpam support or a helper executable should be built and +installed into an appropriate directory. + +It is recommended to use builtin libpam support if you are using +PAM as an intermediate for authentication provider not directly +supported by maddy. + +If PAM authentication requires privileged access on the host system +(e.g. pam_unix.so aka /etc/shadow) then it is recommended to use +a privileged helper executable since maddy process itself won't +have access to it. + +## Built-in PAM support + +Binary artifacts provided for releases do not come with +libpam support. You should build maddy from source. + +See [here](../building-from-source) for detailed instructions. + +You should have libpam development files installed (`libpam-dev` +package on Ubuntu/Debian). + +Then add `--tags 'libpam'` to the build command: +``` +./build.sh --tags 'libpam' +``` + +Then you should be able to replace `local_authdb` implementation +in default configuration with `auth.pam`: +``` +auth.pam local_authdb { + use_helper no +} +``` + +## Helper executable + +TL;DR +``` +git clone https://github.com/foxcpp/maddy +cd maddy/cmd/maddy-pam-helper +gcc pam.c main.c -lpam -o maddy-pam-helper +``` + +Copy the resulting executable into /usr/lib/maddy/ and make +it setuid-root so it can read /etc/shadow (if that's necessary): +``` +chown root:maddy /usr/lib/maddy/maddy-pam-helper +chmod u+xs,g+x,o-x /usr/lib/maddy/maddy-pam-helper +``` + +Then you should be able to replace `local_authdb` implementation +in default configuration with `auth.pam`: +``` +auth.pam local_authdb { + use_helper yes +} +``` + +## Account names + +Since PAM does not use emails for authentication you should configure +maddy to either strip domain part when checking credentials or do not +use email when authenticating. + +See [Multiple domains configuration](/multiple-domains) for how to configure +authentication. + +## PAM service + +You should create a PAM configuration file for maddy to use. +Place it into /etc/pam.d/maddy. +Here is the minimal example using pam_unix (shadow database). +``` +#%PAM-1.0 +auth required pam_unix.so +account required pam_unix.so +``` + +Here is the configuration example you could use on Ubuntu +to use the authentication config system itself uses: +``` +#%PAM-1.0 + +@include common-auth +@include common-account +@include common-session +``` diff --git a/docs/tutorials/setting-up.md b/docs/tutorials/setting-up.md new file mode 100644 index 0000000..2ec8351 --- /dev/null +++ b/docs/tutorials/setting-up.md @@ -0,0 +1,259 @@ +# Installation & initial configuration + +This is the practical guide on how to set up a mail server using maddy for +personal use. It omits most of the technical details for brevity and just gives +you the minimal list of things you need to be aware of and what to do to make +stuff work. + +For purposes of clarity, these values are used in this tutorial as examples, +wherever you see them, you need to replace them with your actual values: + +- Domain: example.org +- MX domain (hostname): mx1.example.org +- IPv4 address: 10.2.3.4 +- IPv6 address: 2001:beef::1 + +## Getting a server + +Where to get a server to run maddy on is out of the scope of this article. Any +VPS (virtual private server) will work fine for small configurations. However, +there are a few things to keep in mind: + +- Make sure your provider does not block SMTP traffic (25 TCP port). Most VPS + providers don't do it, but some "cloud" providers (such as Google Cloud) do + it, so you can't host your mail there. + +- It is recommended to run your own DNS resolver with DNSSEC verification + enabled. + +## Installing maddy + +Your options are: + +* Pre-built tarball (Linux, amd64) + + Available on [GitHub](https://github.com/foxcpp/maddy/releases) or + [maddy.email/builds](https://maddy.email/builds/). + + The tarball includes maddy executable you can + copy into /usr/local/bin as well as systemd unit file you can + use on systemd-based distributions for automatic startup and service + supervision. You should also create "maddy" user and group. + See below for more detailed instructions. + +* Docker image (Linux, amd64) + + ``` + docker pull foxcpp/maddy:0.6 + ``` + + See [here](../../docker) for Docker-specific instructions. + +* Building from source + + See [here](../building-from-source) for instructions. + +* Arch Linux packages + + For Arch Linux users, `maddy` and `maddy-git` PKGBUILDs are available + in AUR. Additionally, binary packages are available in 3rd-party + repository at [https://maddy.email/archlinux/](https://maddy.email/archlinux/) + +## System configuration (systemd-based distribution) + +If you built maddy from source and used `./build.sh install` then +systemd unit files should be already installed. If you used +a pre-built tarball - copy `systemd/*.service` to `/etc/systemd/system` +manually. + +You need to reload service manager configuration to make service available: + +``` +systemctl daemon-reload +``` + +Additionally, you should create maddy user and group. Unlike most other +Linux mail servers, maddy never runs as root. + +``` +useradd -mrU -s /sbin/nologin -d /var/lib/maddy -c "maddy mail server" maddy +``` + +## Host name + domain + +Open /etc/maddy/maddy.conf with vim^W your favorite editor and change +the following lines to match your server name and domain you want to handle +mail for. +If you setup a very small mail server you can use example.org in both fields. +However, to easier a future migration of service, it's recommended to use a +separate DNS entry for that purpose. It's usually mx1.example.org, mx2, etc. +You can of course use another subdomain, for instance: smtp1.example.org. +An email failover server will become possible if you forward mx2.example.org +to another server (as long as you configure it to handle your domain). + +``` +$(hostname) = mx1.example.org +$(primary_domain) = example.org +``` + +If you want to handle multiple domains, you still need to designate +one as "primary". Add all other domains to the `local_domains` line: + +``` +$(local_domains) = $(primary_domain) example.com other.example.com +``` + +## TLS certificates + +One thing that can't be automagically configured is TLS certs. If you already +have them somewhere - use them, open /etc/maddy/maddy.conf and put the right +paths in. You need to make sure maddy can read them while running as +unprivileged user (maddy never runs as root, even during start-up), one way to +do so is to use ACLs (replace with your actual paths): +``` +$ sudo setfacl -R -m u:maddy:rX /etc/ssl/mx1.example.org.crt /etc/ssl/mx1.example.org.key +``` + +maddy reloads TLS certificates from disk once in a minute so it will notice +renewal. It is possible to force reload via `systemctl reload maddy` (or just +`killall -USR2 maddy`). + +### Let's Encrypt and certbot + +If you use certbot to manage your certificates, you can simply symlink +/etc/maddy/certs into /etc/letsencrypt/live. maddy will pick the right +certificate depending on the domain you specified during installation. + +You still need to make keys readable for maddy, though: +``` +$ sudo setfacl -R -m u:maddy:rX /etc/letsencrypt/{live,archive} +``` + +### ACME.sh + +If you use acme.sh to manage your certificates, you could simply run: + +``` +mkdir -p /etc/maddy/certs/mx1.example.org +acme.sh --force --install-cert -d mx1.example.org \ + --key-file /etc/maddy/certs/mx1.example.org/privkey.pem \ + --fullchain-file /etc/maddy/certs/mx1.example.org/fullchain.pem +``` + +## First run + +``` +systemctl start maddy +``` + +The daemon should be running now, except that it is useless because we haven't +configured DNS records. + +## DNS records + +How it is configured depends on your DNS provider (or server, if you run your +own). Here is how your DNS zone should look like: +``` +; Basic domain->IP records, you probably already have them. +example.org. A 10.2.3.4 +example.org. AAAA 2001:beef::1 + +; It says that "server mx1.example.org is handling messages for example.org". +example.org. MX 10 mx1.example.org. +; Of course, mx1 should have A/AAAA entry as well: +mx1.example.org. A 10.2.3.4 +mx1.example.org. AAAA 2001:beef::1 + +; Use SPF to say that the servers in "MX" above are allowed to send email +; for this domain, and nobody else. +example.org. TXT "v=spf1 mx ~all" +; It is recommended to server SPF record for both domain and MX hostname +mx1.example.org. TXT "v=spf1 a ~all" + +; Opt-in into DMARC with permissive policy and request reports about broken +; messages. +_dmarc.example.org. TXT "v=DMARC1; p=quarantine; ruf=mailto:postmaster@example.org" + +; Mark domain as MTA-STS compatible (see the next section) +; and request reports about failures to be sent to postmaster@example.org +_mta-sts.example.org. TXT "v=STSv1; id=1" +_smtp._tls.example.org. TXT "v=TLSRPTv1;rua=mailto:postmaster@example.org" +``` + +And the last one, DKIM key, is a bit tricky. maddy generated a key for you on +the first start-up. You can find it in +/var/lib/maddy/dkim_keys/example.org_default.dns. You need to put it in a TXT +record for `default._domainkey.example.org.` domain, like that: +``` +default._domainkey.example.org. TXT "v=DKIM1; k=ed25519; p=nAcUUozPlhc4VPhp7hZl+owES7j7OlEv0laaDEDBAqg=" +``` + +## MTA-STS and DANE + +By default SMTP is not protected against active attacks. MTA-STS policy tells +compatible senders to always use properly authenticated TLS when talking to +your server, offering a simple-to-deploy way to protect your server against +MitM attacks on port 25. + +Basically, you to create a file with following contents and make it available +at https://mta-sts.example.org/.well-known/mta-sts.txt: +``` +version: STSv1 +mode: enforce +max_age: 604800 +mx: mx1.example.org +``` + +**Note**: mx1.example.org in the file is your MX hostname, In a simple configuration, +it will be the same as your hostname example.org. +In a more complex setups, you would have multiple MX servers - add them all once +per line, like that: + +``` +mx: mx1.example.org +mx: mx2.example.org +``` + +It is also recommended to set a TLSA (DANE) record. +Use https://www.huque.com/bin/gen_tlsa to generate one. +Set port to 25, Transport Protocol to "tcp" and Domain Name to **the MX hostname**. +Example of a valid record: +``` +_25._tcp.mx1.example.org. TLSA 3 1 1 7f59d873a70e224b184c95a4eb54caa9621e47d48b4a25d312d83d96e3498238 +``` + +## User accounts and maddy command + +A mail server is useless without mailboxes, right? Unlike software like postfix +and dovecot, maddy uses "virtual users" by default, meaning it does not care or +know about system users. + +IMAP mailboxes ("accounts") and authentication credentials are kept separate. + +To register user credentials, use `maddy creds create` command. +Like that: +``` +$ maddy creds create postmaster@example.org +``` + +Note the username is a e-mail address. This is required as username is used to +authorize IMAP and SMTP access (unless you configure custom mappings, not +described here). + +After registering the user credentials, you also need to create a local +storage account: +``` +$ maddy imap-acct create postmaster@example.org +``` + +Note: to run `maddy` CLI commands, your user should be in the `maddy` +group. Alternatively, just use `sudo -u maddy`. + +That is it. Now you have your first e-mail address. when authenticating using +your e-mail client, do not forget the username is "postmaster@example.org", not +just "postmaster". + +You may find running `maddy creds --help` and `maddy imap-acct --help` +useful to learn about other commands. Note that IMAP accounts and credentials +are managed separately yet usernames should match by default for things to +work. diff --git a/docs/upgrading.md b/docs/upgrading.md new file mode 100644 index 0000000..88fe4c7 --- /dev/null +++ b/docs/upgrading.md @@ -0,0 +1,117 @@ +# Upgrading from older maddy versions + +It is generally possible to just install latest version (e.g. using build.sh +script) over the existing installation. + +It is recommended to backup state directory (usually /var/lib/maddy for Linux) +before doing so. The new server version may automatically convert DB files in a +way that will make them unreadable by older versions. + +Specific instructions for upgrading between versions with incompatible changes +are documented on this page below. + +## Incompatible version migration + +## 0.2 -> 0.3 + +0.3 includes a significant change to the authentication code that makes it +completely independent of IMAP index. This means 0.2 "unified" database cannot +be used in 0.3 and auto-migration is not possible. Additionally, the way +passwords are hashed is changed, meaning that after migration passwords will +need to be reset. + +**Migration utility is SQLite-specific, if you need one that works for +Postgres - reach out at the IRC channel.** + +1. Make sure the server is not running. + +``` +systemctl stop maddy +``` + +2. Take a backup of `imapsql.db*` files in state directory (/var/lib/maddy). + +``` +mkdir backup +cp /var/lib/maddy/imapsql.db* backup/ +``` + +3. Compile migration utility: + +``` +git clone https://github.com/foxcpp/maddy.git +cd maddy/ +git checkout v0.3.0 +cd cmd/migrate-db-0.2 +go build +``` + +4. Run compiled binary: + +``` +./migrate-db-0.2 /var/lib/maddy/imapsql.db +``` + +5. Open maddy.conf and make following changes: + +Remove `local_authdb` name from imapsql configuration block: +``` +imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db +} +``` + +Add `local_authdb` configuration block using `pass_table` module: + +``` +pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } +} +``` + +6. Use `maddy creds create ACCOUNT_NAME` to add credentials to `pass_table` + store. + +7. Start the server back. + +``` +systemctl start maddy +``` + +## 0.1 -> 0.2 + +0.2 requires several changes in configuration file. + +Change +``` +sql local_mailboxes local_authdb { +``` +to +``` +imapsql local_mailboxes local_authdb { +``` + +Replace +``` +replace_rcpt postmaster postmaster@$(primary_domain) +``` +with +``` +replace_rcpt static { + entry postmaster postmaster@$(primary_domain) +} +``` +and + +``` +replace_rcpt "(.+)\+(.+)@(.+)" "$1@$3" +``` +with +``` +replace_rcpt regexp "(.+)\+(.+)@(.+)" "$1@$3" +``` diff --git a/framework/address/doc.go b/framework/address/doc.go new file mode 100644 index 0000000..3478a3d --- /dev/null +++ b/framework/address/doc.go @@ -0,0 +1,21 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package address provides utilities for parsing +// and validation of RFC 2821 addresses. +package address diff --git a/framework/address/norm.go b/framework/address/norm.go new file mode 100644 index 0000000..6998e66 --- /dev/null +++ b/framework/address/norm.go @@ -0,0 +1,169 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package address + +import ( + "fmt" + "strings" + "unicode/utf8" + + "github.com/foxcpp/maddy/framework/dns" + "golang.org/x/net/idna" + "golang.org/x/text/secure/precis" + "golang.org/x/text/unicode/norm" +) + +// ForLookup transforms the local-part of the address into a canonical form +// usable for map lookups or direct comparisons. +// +// If Equal(addr1, addr2) == true, then ForLookup(addr1) == ForLookup(addr2). +// +// On error, case-folded addr is also returned. +func ForLookup(addr string) (string, error) { + if addr == "" { // Null return-path case. + return "", nil + } + + mbox, domain, err := Split(addr) + if err != nil { + return strings.ToLower(addr), err + } + + if domain != "" { + domain, err = dns.ForLookup(domain) + if err != nil { + return strings.ToLower(addr), err + } + } + + mbox = strings.ToLower(norm.NFC.String(mbox)) + + if domain == "" { + return mbox, nil + } + + return mbox + "@" + domain, nil +} + +// CleanDomain returns the address with the domain part converted into its canonical form. +// +// More specifically, converts the domain part of the address to U-labels, +// normalizes it to NFC and then case-folds it. +// +// Original value is also returned on the error. +func CleanDomain(addr string) (string, error) { + if addr == "" { // Null return-path + return "", nil + } + + mbox, domain, err := Split(addr) + if err != nil { + return addr, err + } + + uDomain, err := idna.ToUnicode(domain) + if err != nil { + return addr, err + } + uDomain = strings.ToLower(norm.NFC.String(uDomain)) + + if domain == "" { + return mbox, nil + } + + return mbox + "@" + uDomain, nil +} + +// Equal reports whether addr1 and addr2 are considered to be +// case-insensitively equivalent. +// +// The equivalence is defined to be the conjunction of IDN label equivalence +// for the domain part and canonical equivalence* of the local-part converted +// to lower case. +// +// * IDN label equivalence is defined by RFC 5890 Section 2.3.2.4. +// ** Canonical equivalence is defined by UAX #15. +// +// Equivalence for malformed addresses is defined using regular byte-string +// comparison with case-folding applied. +func Equal(addr1, addr2 string) bool { + // Short circuit. If they are bit-equivalent, then they are also canonically + // equivalent. + if addr1 == addr2 { + return true + } + + uAddr1, _ := ForLookup(addr1) + uAddr2, _ := ForLookup(addr2) + return uAddr1 == uAddr2 +} + +func IsASCII(s string) bool { + for _, ch := range s { + if ch > utf8.RuneSelf { + return false + } + } + return true +} + +func FQDNDomain(addr string) string { + if strings.HasSuffix(addr, ".") { + return addr + } + return addr + "." +} + +// PRECISFold applies UsernameCaseMapped to the local part and dns.ForLookup +// to domain part of the address. +func PRECISFold(addr string) (string, error) { + return precisEmail(addr, precis.UsernameCaseMapped) +} + +// PRECIS applies UsernameCasePreserved to the local part and dns.ForLookup +// to domain part of the address. +func PRECIS(addr string) (string, error) { + return precisEmail(addr, precis.UsernameCasePreserved) +} + +func precisEmail(addr string, profile *precis.Profile) (string, error) { + mbox, domain, err := Split(addr) + if err != nil { + return "", fmt.Errorf("address: precis: %w", err) + } + + // PRECISFold is not included in the regular address.ForLookup since it reduces + // the range of valid addresses to a subset of actually valid values. + // PRECISFold is a matter of our own local policy, not a general rule for all + // email addresses. + + // Side note: For used profiles, there is no practical difference between + // CompareKey and String. + mbox, err = profile.CompareKey(mbox) + if err != nil { + return "", fmt.Errorf("address: precis: %w", err) + } + + domain, err = dns.ForLookup(domain) + if err != nil { + return "", fmt.Errorf("address: precis: %w", err) + } + + return mbox + "@" + domain, nil +} diff --git a/framework/address/norm_test.go b/framework/address/norm_test.go new file mode 100644 index 0000000..1b6d511 --- /dev/null +++ b/framework/address/norm_test.go @@ -0,0 +1,88 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package address + +import ( + "testing" +) + +func addrFuncTest(t *testing.T, f func(string) (string, error)) func(in, wantOut string, fail bool) { + return func(in, wantOut string, fail bool) { + t.Helper() + + out, err := f(in) + if err != nil { + if !fail { + t.Errorf("Expected failure, got none") + } + } + if out != wantOut { + t.Errorf("Wrong result: want '%s', got '%s'", wantOut, out) + } + } +} + +func TestForLookup(t *testing.T) { + test := addrFuncTest(t, ForLookup) + test("test@example.org", "test@example.org", false) + test("E\u0301@example.org", "\u00E9@example.org", false) + test("test@EXAMPLE.org", "test@example.org", false) + test("test@xn--e1aybc.example.org", "test@тест.example.org", false) + test("TEST@xn--99999999999.example.org", "test@xn--99999999999.example.org", true) + test("tESt@", "test@", true) + test("postmaster", "postmaster", false) +} + +func TestCleanDomain(t *testing.T) { + test := addrFuncTest(t, CleanDomain) + test("test@example.org", "test@example.org", false) + test("whateveR@example.org", "whateveR@example.org", false) + test("E\u0301@example.org", "E\u0301@example.org", false) + test("test@EXAMPLE.org", "test@example.org", false) + test("test@xn--e1aybc.example.org", "test@тест.example.org", false) + test("TEST@xn--99999999999.example.org", "TEST@xn--99999999999.example.org", true) + test("tESt@", "tESt@", true) + test("postmaster", "postmaster", false) +} + +func TestEqual(t *testing.T) { + test := func(in1, in2 string, wantEq bool) { + eq := Equal(in1, in2) + if eq != wantEq { + t.Errorf("Want Equal(%s, %s) == %v, got %v", in1, in2, wantEq, eq) + } + } + + test("test@example.org", "test@example.org", true) + test("test2@example.org", "test@example.org", false) + test("TEST2@example.org", "TesT2@example.org", true) + test("E\u0301@example.org", "\u00E9@example.org", true) + test("test@тест.example.org", "test@xn--e1aybc.example.org", true) + test("test@xn--999999999999999.example.org", "test@xn--999999999999999.example.org", true) + test("test@xn--999999999999.example.org", "test@xn--999999999999999.example.org", false) +} + +func TestIsASCII(t *testing.T) { + if !IsASCII("hello") { + t.Errorf("'hello' is ASCII") + } + if IsASCII("тест") { + t.Errorf("'тест' is non-ASCII") + } +} diff --git a/framework/address/rfc6531.go b/framework/address/rfc6531.go new file mode 100644 index 0000000..dfa5420 --- /dev/null +++ b/framework/address/rfc6531.go @@ -0,0 +1,85 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package address + +import ( + "errors" + + "golang.org/x/net/idna" + "golang.org/x/text/unicode/norm" +) + +var ErrUnicodeMailbox = errors.New("address: cannot convert the Unicode local-part to the ACE form") + +// ToASCII converts the domain part of the email address to the A-label form and +// fails with ErrUnicodeMailbox if the local-part contains non-ASCII characters. +func ToASCII(addr string) (string, error) { + mbox, domain, err := Split(addr) + if err != nil { + return addr, err + } + + for _, ch := range mbox { + if ch > 128 { + return addr, ErrUnicodeMailbox + } + } + + if domain == "" { + return mbox, nil + } + + aDomain, err := idna.ToASCII(domain) + if err != nil { + return addr, err + } + + return mbox + "@" + aDomain, nil +} + +// ToUnicode converts the domain part of the email address to the U-label form. +func ToUnicode(addr string) (string, error) { + mbox, domain, err := Split(addr) + if err != nil { + return norm.NFC.String(addr), err + } + + if domain == "" { + return mbox, nil + } + + uDomain, err := idna.ToUnicode(domain) + if err != nil { + return norm.NFC.String(addr), err + } + + return mbox + "@" + norm.NFC.String(uDomain), nil +} + +// SelectIDNA is a convenience function for conversion of domains in the email +// addresses to/from the Punycode form. +// +// ulabel=true => ToUnicode is used. +// ulabel=false => ToASCII is used. +func SelectIDNA(ulabel bool, addr string) (string, error) { + if ulabel { + return ToUnicode(addr) + } + return ToASCII(addr) +} diff --git a/framework/address/rfc6531_test.go b/framework/address/rfc6531_test.go new file mode 100644 index 0000000..f6cec25 --- /dev/null +++ b/framework/address/rfc6531_test.go @@ -0,0 +1,41 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package address + +import ( + "strings" + "testing" +) + +func TestToASCII(t *testing.T) { + test := addrFuncTest(t, ToASCII) + test("test@тест.example.org", "test@xn--e1aybc.example.org", false) + test("test@org."+strings.Repeat("x", 65535)+"\uFF00", "test@org."+strings.Repeat("x", 65535)+"\uFF00", true) + test("тест@example.org", "тест@example.org", true) + test("postmaster", "postmaster", false) + test("postmaster@", "postmaster@", true) +} + +func TestToUnicode(t *testing.T) { + test := addrFuncTest(t, ToUnicode) + test("test@xn--e1aybc.example.org", "test@тест.example.org", false) + test("test@xn--9999999999999999999a.org", "test@xn--9999999999999999999a.org", true) + test("postmaster", "postmaster", false) + test("postmaster@", "postmaster@", true) +} diff --git a/framework/address/split.go b/framework/address/split.go new file mode 100644 index 0000000..88b8d51 --- /dev/null +++ b/framework/address/split.go @@ -0,0 +1,132 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package address + +import ( + "errors" + "strings" +) + +// Split splits a email address (as defined by RFC 5321 as a forward-path +// token) into local part (mailbox) and domain. +// +// Note that definition of the forward-path token includes the special +// postmaster address without the domain part. Split will return domain == "" +// in this case. +// +// Split does almost no sanity checks on the input and is intentionally naive. +// If this is a concern, ValidMailbox and ValidDomain should be used on the +// output. +func Split(addr string) (mailbox, domain string, err error) { + if strings.EqualFold(addr, "postmaster") { + return addr, "", nil + } + + indx := strings.LastIndexByte(addr, '@') + if indx == -1 { + return "", "", errors.New("address: missing at-sign") + } + mailbox = addr[:indx] + domain = addr[indx+1:] + if mailbox == "" { + return "", "", errors.New("address: empty local-part") + } + if domain == "" { + return "", "", errors.New("address: empty domain") + } + return +} + +// UnquoteMbox undoes escaping and quoting of the local-part. That is, for +// local-part `"test\" @ test"` it will return `test" @test`. +func UnquoteMbox(mbox string) (string, error) { + var ( + quoted bool + escaped bool + terminatedQuote bool + mailboxB strings.Builder + ) + for _, ch := range mbox { + if terminatedQuote { + return "", errors.New("address: closing quote should be right before at-sign") + } + + switch ch { + case '"': + if !escaped { + quoted = !quoted + if !quoted { + terminatedQuote = true + } + continue + } + case '\\': + if !escaped { + if !quoted { + return "", errors.New("address: escapes are allowed only in quoted strings") + } + escaped = true + continue + } + case '@': + if !quoted { + return "", errors.New("address: extra at-sign in non-quoted local-part") + } + } + + escaped = false + + mailboxB.WriteRune(ch) + } + + if mailboxB.Len() == 0 { + return "", errors.New("address: empty local part") + } + + return mailboxB.String(), nil +} + +// "specials" from RFC5322 grammar with dot removed (it is defined in grammar separately, for some reason) +var mboxSpecial = map[rune]struct{}{ + '(': {}, ')': {}, '<': {}, '>': {}, + '[': {}, ']': {}, ':': {}, ';': {}, + '@': {}, '\\': {}, ',': {}, + '"': {}, ' ': {}, +} + +func QuoteMbox(mbox string) string { + var mailboxEsc strings.Builder + mailboxEsc.Grow(len(mbox)) + quoted := false + for _, ch := range mbox { + if _, ok := mboxSpecial[ch]; ok { + if ch == '\\' || ch == '"' { + mailboxEsc.WriteRune('\\') + } + mailboxEsc.WriteRune(ch) + quoted = true + } else { + mailboxEsc.WriteRune(ch) + } + } + if quoted { + return `"` + mailboxEsc.String() + `"` + } + return mbox +} diff --git a/framework/address/split_test.go b/framework/address/split_test.go new file mode 100644 index 0000000..b5a8df5 --- /dev/null +++ b/framework/address/split_test.go @@ -0,0 +1,110 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package address + +import ( + "testing" +) + +func TestSplit(t *testing.T) { + test := func(addr, mbox, domain string, fail bool) { + t.Helper() + + actualMbox, actualDomain, err := Split(addr) + if err != nil && !fail { + t.Errorf("%s: unexpected error: %v", addr, err) + return + } + if err == nil && fail { + t.Errorf("%s: expected error, got %s, %s", addr, actualMbox, actualDomain) + return + } + + if actualMbox != mbox { + t.Errorf("%s: wrong local part, want %s, got %s", addr, mbox, actualMbox) + } + if actualDomain != domain { + t.Errorf("%s: wrong domain part, want %s, got %s", addr, domain, actualDomain) + } + } + + test("simple@example.org", "simple", "example.org", false) + test("simple@[1.2.3.4]", "simple", "[1.2.3.4]", false) + test("simple@[IPv6:beef::1]", "simple", "[IPv6:beef::1]", false) + test("@example.org", "", "", true) + test("@", "", "", true) + test("no-domain@", "", "", true) + test("@no-local-part", "", "", true) + + // Not a valid address, but a special value for SMTP + // should be handled separately where necessary. + test("", "", "", true) + + // A special SMTP value too, but permitted now. + test("postmaster", "postmaster", "", false) +} + +func TestUnquoteMbox(t *testing.T) { + test := func(inputMbox, expectedMbox string, fail bool) { + t.Helper() + + actualMbox, err := UnquoteMbox(inputMbox) + if err != nil && !fail { + t.Errorf("unexpected error: %v", err) + return + } + if err == nil && fail { + t.Errorf("expected error, got %s", actualMbox) + return + } + + if actualMbox != expectedMbox { + t.Errorf("wrong local part, want %s, got %s", actualMbox, actualMbox) + } + } + + test(`no\@no`, "", true) + test("no@no", "", true) + test(`no\"no`, "", true) + test(`"no\"no"`, `no"no`, false) + test(`"no@no"`, `no@no`, false) + test(`"no no"`, `no no`, false) + test(`"no\\no"`, `no\no`, false) + test(`"no"no`, "", true) + test(`postmaster`, "postmaster", false) + test(`foo`, "foo", false) +} + +func TestQuoteMbox(t *testing.T) { + test := func(inputMbox, expectedMbox string) { + t.Helper() + + actualMbox := QuoteMbox(inputMbox) + if actualMbox != expectedMbox { + t.Errorf("wrong local part, want %s, got %s", actualMbox, actualMbox) + } + } + + test(`no"no`, `"no\"no"`) + test(`no@no`, `"no@no"`) + test(`no no`, `"no no"`) + test(`no\no`, `"no\\no"`) + test("postmaster", `postmaster`) + test("foo", `foo`) +} diff --git a/framework/address/validation.go b/framework/address/validation.go new file mode 100644 index 0000000..a165adc --- /dev/null +++ b/framework/address/validation.go @@ -0,0 +1,138 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package address + +import ( + "strings" + + "golang.org/x/net/idna" +) + +/* +Rules for validation are subset of rules listed here: +https://emailregex.com/email-validation-summary/ +*/ + +// Valid checks whether ths string is valid as a email address as defined by +// RFC 5321. +func Valid(addr string) bool { + if len(addr) > 320 { // RFC 3696 says it's 320, not 255. + return false + } + + mbox, domain, err := Split(addr) + if err != nil { + return false + } + + // The only case where this can be true is "postmaster". + // So allow it. + if domain == "" { + return true + } + + return ValidMailboxName(mbox) && ValidDomain(domain) +} + +var validGraphic = map[rune]bool{ + '!': true, '#': true, + '$': true, '%': true, + '&': true, '\'': true, + '*': true, '+': true, + '-': true, '/': true, + '=': true, '?': true, + '^': true, '_': true, + '`': true, '{': true, + '|': true, '}': true, + '~': true, '.': true, +} + +// ValidMailboxName checks whether the specified string is a valid mailbox-name +// element of e-mail address (left part of it, before at-sign). +func ValidMailboxName(mbox string) bool { + if strings.HasPrefix(mbox, `"`) { + raw, err := UnquoteMbox(mbox) + if err != nil { + return false + } + + // Inside quotes, any ASCII graphic and space is allowed. + // Additionally, RFC 6531 extends that to allow any Unicode (UTF-8). + for _, ch := range raw { + if ch < ' ' || ch == 0x7F /* DEL */ { + // ASCII control characters. + return false + } + } + return true + } + + // Without quotes, limited set of ASCII graphics is allowed + ASCII + // alphanumeric characters. + // RFC 6531 extends that to allow any Unicode (UTF-8). + for _, ch := range mbox { + if validGraphic[ch] { + continue + } + if ch >= '0' && ch <= '9' { + continue + } + if ch >= 'A' && ch <= 'Z' { + continue + } + if ch >= 'a' && ch <= 'z' { + continue + } + if ch > 0x7F { // Unicode + continue + } + + return false + } + + return true +} + +// ValidDomain checks whether the specified string is a valid DNS domain. +func ValidDomain(domain string) bool { + if len(domain) > 255 || len(domain) == 0 { + return false + } + if strings.HasPrefix(domain, ".") { + return false + } + if strings.Contains(domain, "..") { + return false + } + + // Length checks are to be applied to A-labels form. + // maddy uses U-labels representation across the code (for lookups, etc). + domainASCII, err := idna.ToASCII(domain) + if err != nil { + return false + } + labels := strings.Split(domainASCII, ".") + for _, label := range labels { + if len(label) > 64 { + return false + } + } + + return true +} diff --git a/framework/address/validation_test.go b/framework/address/validation_test.go new file mode 100644 index 0000000..fdac834 --- /dev/null +++ b/framework/address/validation_test.go @@ -0,0 +1,33 @@ +package address_test + +import ( + "strings" + "testing" + + "github.com/foxcpp/maddy/framework/address" +) + +func TestValidMailboxName(t *testing.T) { + if !address.ValidMailboxName("caddy.bug") { + t.Error("caddy.bug should be valid mailbox name") + } +} + +func TestValidDomain(t *testing.T) { + for _, c := range []struct { + Domain string + Valid bool + }{ + {Domain: "maddy.email", Valid: true}, + {Domain: "", Valid: false}, + {Domain: "maddy.email.", Valid: true}, + {Domain: "..", Valid: false}, + {Domain: strings.Repeat("a", 256), Valid: false}, + {Domain: "äõäoaõoäaõaäõaoäaoaäõoaäooaoaoiuaiauäõiuüõaõäiauõaaa.tld", Valid: true}, // https://github.com/foxcpp/maddy/issues/554 + {Domain: "xn--oaoaaaoaoaoaooaoaoiuaiauiuaiauaaa-f1cadccdcmd01eddchqcbe07a.tld", Valid: true}, // https://github.com/foxcpp/maddy/issues/554 + } { + if actual := address.ValidDomain(c.Domain); actual != c.Valid { + t.Errorf("expected domain %v to be valid=%v, but got %v", c.Domain, c.Valid, actual) + } + } +} diff --git a/framework/buffer/buffer.go b/framework/buffer/buffer.go new file mode 100644 index 0000000..e386046 --- /dev/null +++ b/framework/buffer/buffer.go @@ -0,0 +1,60 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// The buffer package provides utilities for temporary storage (buffering) +// of large blobs. +package buffer + +import ( + "io" +) + +// Buffer interface represents abstract temporary storage for blobs. +// +// The Buffer storage is assumed to be immutable. If any modifications +// are made - new storage location should be used for them. +// This is important to ensure goroutine-safety. +// +// Since Buffer objects require a careful management of lifetimes, here +// is the convention: Its always creator responsibility to call Remove after +// Buffer is no longer used. If Buffer object is passed to a function - it is not +// guaranteed to be valid after this function returns. If function needs to preserve +// the storage contents, it should "re-buffer" it either by reading entire blob +// and storing it somewhere or applying implementation-specific methods (for example, +// the FileBuffer storage may be "re-buffered" by hard-linking the underlying file). +type Buffer interface { + // Open creates new Reader reading from the underlying storage. + Open() (io.ReadCloser, error) + + // Len reports the length of the stored blob. + // + // Notably, it indicates the amount of bytes that can be read from the + // newly created Reader without hiting io.EOF. + Len() int + + // Remove discards buffered body and releases all associated resources. + // + // Multiple Buffer objects may refer to the same underlying storage. + // In this case, care should be taken to ensure that Remove is called + // only once since it will discard the shared storage and invalidate + // all Buffer objects using it. + // + // Readers previously created using Open can still be used, but + // new ones can't be created. + Remove() error +} diff --git a/framework/buffer/bytesreader.go b/framework/buffer/bytesreader.go new file mode 100644 index 0000000..365a75f --- /dev/null +++ b/framework/buffer/bytesreader.go @@ -0,0 +1,61 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package buffer + +import ( + "bytes" +) + +// BytesReader is a wrapper for bytes.Reader that stores the original []byte +// value and allows to retrieve it. +// +// It is meant for passing to libraries that expect a io.Reader +// but apply certain optimizations when the Reader implements +// Bytes() interface. +type BytesReader struct { + *bytes.Reader + value []byte +} + +// Bytes returns the unread portion of underlying slice used to construct +// BytesReader. +func (br BytesReader) Bytes() []byte { + return br.value[int(br.Size())-br.Len():] +} + +// Copy returns the BytesReader reading from the same slice as br at the same +// position. +func (br BytesReader) Copy() BytesReader { + return NewBytesReader(br.Bytes()) +} + +// Close is a dummy method for implementation of io.Closer so BytesReader can +// be used in MemoryBuffer directly. +func (br BytesReader) Close() error { + return nil +} + +func NewBytesReader(b []byte) BytesReader { + // BytesReader and not *BytesReader because BytesReader already wraps two + // pointers and double indirection would be pointless. + return BytesReader{ + Reader: bytes.NewReader(b), + value: b, + } +} diff --git a/framework/buffer/file.go b/framework/buffer/file.go new file mode 100644 index 0000000..0025984 --- /dev/null +++ b/framework/buffer/file.go @@ -0,0 +1,85 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package buffer + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" +) + +// FileBuffer implements Buffer interface using file system. +type FileBuffer struct { + Path string + + // LenHint is the size of the stored blob. It can + // be set to avoid the need to call os.Stat in the + // Len() method. + LenHint int +} + +func (fb FileBuffer) Open() (io.ReadCloser, error) { + return os.Open(fb.Path) +} + +func (fb FileBuffer) Len() int { + if fb.LenHint != 0 { + return fb.LenHint + } + + info, err := os.Stat(fb.Path) + if err != nil { + // Any access to the file will probably fail too. So we can't return a + // sensible value. + return 0 + } + + return int(info.Size()) +} + +func (fb FileBuffer) Remove() error { + return os.Remove(fb.Path) +} + +// BufferInFile is a convenience function which creates FileBuffer with underlying +// file created in the specified directory with the random name. +func BufferInFile(r io.Reader, dir string) (Buffer, error) { + // It is assumed that PRNG is initialized somewhere during program startup. + nameBytes := make([]byte, 32) + _, err := rand.Read(nameBytes) + if err != nil { + return nil, fmt.Errorf("buffer: failed to generate randomness for file name: %v", err) + } + path := filepath.Join(dir, hex.EncodeToString(nameBytes)) + f, err := os.Create(path) + if err != nil { + return nil, fmt.Errorf("buffer: failed to create file: %v", err) + } + if _, err = io.Copy(f, r); err != nil { + return nil, fmt.Errorf("buffer: failed to write file: %v", err) + } + if err := f.Close(); err != nil { + return nil, fmt.Errorf("buffer: failed to close file: %v", err) + } + + return FileBuffer{Path: path}, nil +} diff --git a/framework/buffer/memory.go b/framework/buffer/memory.go new file mode 100644 index 0000000..dafd677 --- /dev/null +++ b/framework/buffer/memory.go @@ -0,0 +1,50 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package buffer + +import ( + "io" +) + +// MemoryBuffer implements Buffer interface using byte slice. +type MemoryBuffer struct { + Slice []byte +} + +func (mb MemoryBuffer) Open() (io.ReadCloser, error) { + return NewBytesReader(mb.Slice), nil +} + +func (mb MemoryBuffer) Len() int { + return len(mb.Slice) +} + +func (mb MemoryBuffer) Remove() error { + return nil +} + +// BufferInMemory is a convenience function which creates MemoryBuffer with +// contents of the passed io.Reader. +func BufferInMemory(r io.Reader) (Buffer, error) { + blob, err := io.ReadAll(r) + if err != nil { + return nil, err + } + return MemoryBuffer{Slice: blob}, nil +} diff --git a/framework/cfgparser/env.go b/framework/cfgparser/env.go new file mode 100644 index 0000000..c7ad03c --- /dev/null +++ b/framework/cfgparser/env.go @@ -0,0 +1,67 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package parser + +import ( + "os" + "regexp" + "strings" +) + +func expandEnvironment(nodes []Node) []Node { + // If nodes is nil - don't replace with empty slice, as nil indicates "no + // block". + if nodes == nil { + return nil + } + + replacer := buildEnvReplacer() + newNodes := make([]Node, 0, len(nodes)) + for _, node := range nodes { + node.Name = removeUnexpandedEnvvars(replacer.Replace(node.Name)) + newArgs := make([]string, 0, len(node.Args)) + for _, arg := range node.Args { + newArgs = append(newArgs, removeUnexpandedEnvvars(replacer.Replace(arg))) + } + node.Args = newArgs + node.Children = expandEnvironment(node.Children) + newNodes = append(newNodes, node) + } + return newNodes +} + +var unixEnvvarRe = regexp.MustCompile(`{env:([^\$]+)}`) + +func removeUnexpandedEnvvars(s string) string { + s = unixEnvvarRe.ReplaceAllString(s, "") + return s +} + +func buildEnvReplacer() *strings.Replacer { + env := os.Environ() + pairs := make([]string, 0, len(env)*4) + for _, entry := range env { + parts := strings.SplitN(entry, "=", 2) + key := parts[0] + value := parts[1] + + pairs = append(pairs, "{env:"+key+"}", value) + } + return strings.NewReplacer(pairs...) +} diff --git a/framework/cfgparser/imports.go b/framework/cfgparser/imports.go new file mode 100644 index 0000000..8078dd8 --- /dev/null +++ b/framework/cfgparser/imports.go @@ -0,0 +1,176 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package parser + +import ( + "os" + "path/filepath" + "regexp" + "strings" +) + +func (ctx *parseContext) expandImports(node Node, expansionDepth int) (Node, error) { + // Leave nil value as is because it is used as non-existent block indicator + // (vs empty slice - empty block). + if node.Children == nil { + return node, nil + } + + newChildrens := make([]Node, 0, len(node.Children)) + containsImports := false + for _, child := range node.Children { + child, err := ctx.expandImports(child, expansionDepth+1) + if err != nil { + return node, err + } + + if child.Name == "import" { + // We check it here instead of function start so we can + // use line information from import directive that is likely + // caused this error. + if expansionDepth > 255 { + return node, NodeErr(child, "hit import expansion limit") + } + + containsImports = true + if len(child.Args) != 1 { + return node, ctx.Err("import directive requires exactly 1 argument") + } + + subtree, err := ctx.resolveImport(child, child.Args[0], expansionDepth) + if err != nil { + return node, err + } + + newChildrens = append(newChildrens, subtree...) + } else { + newChildrens = append(newChildrens, child) + } + } + node.Children = newChildrens + + // We need to do another pass to expand any imports added by snippets we + // just expanded. + if containsImports { + return ctx.expandImports(node, expansionDepth+1) + } + + return node, nil +} + +func (ctx *parseContext) resolveImport(node Node, name string, expansionDepth int) ([]Node, error) { + if subtree, ok := ctx.snippets[name]; ok { + return subtree, nil + } + + file := name + if !filepath.IsAbs(name) { + file = filepath.Join(filepath.Dir(ctx.fileLocation), name) + } + src, err := os.Open(file) + if err != nil { + if os.IsNotExist(err) { + src, err = os.Open(file + ".conf") + if err != nil { + if os.IsNotExist(err) { + return nil, NodeErr(node, "unknown import: %s", name) + } + return nil, err + } + } else { + return nil, err + } + } + nodes, snips, macros, err := readTree(src, file, expansionDepth+1) + if err != nil { + return nodes, err + } + for k, v := range snips { + ctx.snippets[k] = v + } + for k, v := range macros { + ctx.macros[k] = v + } + + return nodes, nil +} + +func (ctx *parseContext) expandMacros(node *Node) error { + if strings.HasPrefix(node.Name, "$(") && strings.HasSuffix(node.Name, ")") { + return ctx.Err("can't use macro argument as directive name") + } + + newArgs := make([]string, 0, len(node.Args)) + for _, arg := range node.Args { + if !strings.HasPrefix(arg, "$(") || !strings.HasSuffix(arg, ")") { + if strings.Contains(arg, "$(") && strings.Contains(arg, ")") { + var err error + arg, err = ctx.expandSingleValueMacro(arg) + if err != nil { + return err + } + } + + newArgs = append(newArgs, arg) + continue + } + + macroName := arg[2 : len(arg)-1] + replacement, ok := ctx.macros[macroName] + if !ok { + // Undefined macros are expanded to zero arguments. + continue + } + + newArgs = append(newArgs, replacement...) + } + node.Args = newArgs + + if node.Children != nil { + for i := range node.Children { + if err := ctx.expandMacros(&node.Children[i]); err != nil { + return err + } + } + } + + return nil +} + +var macroRe = regexp.MustCompile(`\$\(([^\$]+)\)`) + +func (ctx *parseContext) expandSingleValueMacro(arg string) (string, error) { + matches := macroRe.FindAllStringSubmatch(arg, -1) + for _, match := range matches { + macroName := match[1] + if len(ctx.macros[macroName]) > 1 { + return "", ctx.Err("can't expand macro with multiple arguments inside a string") + } + + var value string + if ctx.macros[macroName] != nil { + // Macros have at least one argument. + value = ctx.macros[macroName][0] + } + + arg = strings.Replace(arg, "$("+macroName+")", value, -1) + } + + return arg, nil +} diff --git a/framework/cfgparser/parse.go b/framework/cfgparser/parse.go new file mode 100644 index 0000000..aed01e3 --- /dev/null +++ b/framework/cfgparser/parse.go @@ -0,0 +1,391 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package config provides set of utilities for configuration parsing. +package parser + +import ( + "errors" + "fmt" + "io" + "strings" + "unicode" + + "github.com/foxcpp/maddy/framework/config/lexer" +) + +// Node struct describes a parsed configurtion block or a simple directive. +// +// name arg0 arg1 { +// children0 +// children1 +// } +type Node struct { + // Name is the first string at node's line. + Name string + // Args are any strings placed after the node name. + Args []string + + // Children slice contains all children blocks if node is a block. Can be nil. + Children []Node + + // Snippet indicates whether current parsed node is a snippet. Always false + // for all nodes returned from Read because snippets are expanded before it + // returns. + Snippet bool + + // Macro indicates whether current parsed node is a macro. Always false + // for all nodes returned from Read because macros are expanded before it + // returns. + Macro bool + + // File is the name of node's source file. + File string + + // Line is the line number where the directive is located in the source file. For + // blocks this is the line where "block header" (name + args) resides. + Line int +} + +type parseContext struct { + lexer.Dispenser + nesting int + snippets map[string][]Node + macros map[string][]string + + fileLocation string +} + +func validateNodeName(s string) error { + if len(s) == 0 { + return errors.New("empty directive name") + } + + if unicode.IsDigit([]rune(s)[0]) { + return errors.New("directive name cannot start with a digit") + } + + allowedPunct := map[rune]bool{'.': true, '-': true, '_': true} + + for _, ch := range s { + if !unicode.IsLetter(ch) && + !unicode.IsDigit(ch) && + !allowedPunct[ch] { + return errors.New("character not allowed in directive name: " + string(ch)) + } + } + + return nil +} + +// readNode reads node starting at current token pointed by the lexer's +// cursor (it should point to node name). +// +// After readNode returns, the lexer's cursor will point to the last token of the parsed +// Node. This ensures predictable cursor location independently of the EOF state. +// Thus code reading multiple nodes should call readNode then manually +// advance lexer cursor (ctx.Next) and either call readNode again or stop +// because cursor hit EOF. +// +// readNode calls readNodes if currently parsed node is a block. +func (ctx *parseContext) readNode() (Node, error) { + node := Node{} + node.File = ctx.File() + node.Line = ctx.Line() + + if ctx.Val() == "{" { + return node, ctx.SyntaxErr("block header") + } + + node.Name = ctx.Val() + if ok, name := ctx.isSnippet(node.Name); ok { + node.Name = name + node.Snippet = true + } + + var continueOnLF bool + for { + for ctx.NextArg() || (continueOnLF && ctx.NextLine()) { + continueOnLF = false + // name arg0 arg1 { + // # ^ called when we hit this token + // c0 + // c1 + // } + if ctx.Val() == "{" { + var err error + node.Children, err = ctx.readNodes() + if err != nil { + return node, err + } + break + } + + node.Args = append(node.Args, ctx.Val()) + } + + // Continue reading the same Node if the \ was used to escape the newline. + // E.g. + // name arg0 arg1 \ + // arg2 arg3 + if len(node.Args) != 0 && node.Args[len(node.Args)-1] == `\` { + last := len(node.Args) - 1 + node.Args[last] = node.Args[last][:len(node.Args[last])-1] + if len(node.Args[last]) == 0 { + node.Args = node.Args[:last] + } + continueOnLF = true + continue + } + break + } + + macroName, macroArgs, err := ctx.parseAsMacro(&node) + if err != nil { + return node, err + } + if macroName != "" { + node.Name = macroName + node.Args = macroArgs + node.Macro = true + } + + if !node.Macro && !node.Snippet { + if err := validateNodeName(node.Name); err != nil { + return node, err + } + } + + return node, nil +} + +func NodeErr(node Node, f string, args ...interface{}) error { + if node.File == "" { + return fmt.Errorf(f, args...) + } + return fmt.Errorf("%s:%d: %s", node.File, node.Line, fmt.Sprintf(f, args...)) +} + +func (ctx *parseContext) isSnippet(name string) (bool, string) { + if strings.HasPrefix(name, "(") && strings.HasSuffix(name, ")") { + return true, name[1 : len(name)-1] + } + return false, "" +} + +func (ctx *parseContext) parseAsMacro(node *Node) (macroName string, args []string, err error) { + if !strings.HasPrefix(node.Name, "$(") { + return "", nil, nil + } + if !strings.HasSuffix(node.Name, ")") { + return "", nil, ctx.Err("macro name must end with )") + } + macroName = node.Name[2 : len(node.Name)-1] + if len(node.Args) < 2 { + return macroName, nil, ctx.Err("at least 2 arguments are required") + } + if node.Args[0] != "=" { + return macroName, nil, ctx.Err("missing = in macro declaration") + } + return macroName, node.Args[1:], nil +} + +// readNodes reads nodes from the currently parsed block. +// +// The lexer's cursor should point to the opening brace +// name arg0 arg1 { #< this one +// +// c0 +// c1 +// } +// +// To stay consistent with readNode after this function returns the lexer's cursor points +// to the last token of the black (closing brace). +func (ctx *parseContext) readNodes() ([]Node, error) { + // It is not 'var res []Node' because we want empty + // but non-nil Children slice for empty braces. + res := []Node{} + + if ctx.nesting > 255 { + return res, ctx.Err("nesting limit reached") + } + + ctx.nesting++ + + var requireNewLine bool + // This loop iterates over logical lines. + // Here are some examples, '#' is placed before token where cursor is when + // another iteration of this loop starts. + // + // #a + // #a b + // #a b { + // #ac aa + // #} + // #aa bbb bbb \ + // ccc ccc + // #a b { #ac aa } + // + // As can be seen by the latest example, sometimes such logical line might + // not be terminated by an actual LF character and so this needs to be + // handled carefully. + // + // Note that if the '}' is on the same physical line, it is currently + // included as the part of the logical line, that is: + // #a b { #ac aa } + // ^------- that's the logical line + // #c d + // ^--- that's the next logical line + // This is handled by the "edge case" branch inside the loop. + for { + if requireNewLine { + if !ctx.NextLine() { + // If we can't advance cursor even without Line constraint - + // that's EOF. + if !ctx.Next() { + return res, nil + } + return res, ctx.Err("newline is required after closing brace") + } + } else if !ctx.Next() { + break + } + + // name arg0 arg1 { + // c0 + // c1 + // } + // ^ called when we hit } on separate line, + // This means block we hit end of our block. + if ctx.Val() == "}" { + ctx.nesting-- + // name arg0 arg1 { #<1 + // } } + // ^2 ^3 + // + // After #1 ctx.nesting is incremented by ctx.nesting++ before this loop. + // Then we advance cursor and hit }, we exit loop, ctx.nesting now becomes 0. + // But then the parent block reader does the same when it hits #3 - + // ctx.nesting becomes -1 and it fails. + if ctx.nesting < 0 { + return res, ctx.Err("unexpected }") + } + break + } + node, err := ctx.readNode() + if err != nil { + return res, err + } + requireNewLine = true + + shouldStop := false + + // name arg0 arg1 { + // c1 c2 } + // ^ + // Edge case, here we check if the last argument of the last node is a } + // If it is - we stop as we hit the end of our block. + if len(node.Args) != 0 && node.Args[len(node.Args)-1] == "}" { + ctx.nesting-- + if ctx.nesting < 0 { + return res, ctx.Err("unexpected }") + } + node.Args = node.Args[:len(node.Args)-1] + shouldStop = true + } + + if node.Macro { + if ctx.nesting != 0 { + return res, ctx.Err("macro declarations are only allowed at top-level") + } + + // Macro declaration itself can contain macro references. + if err := ctx.expandMacros(&node); err != nil { + return res, err + } + + // = sign is removed by parseAsMacro. + // It also cuts $( and ) from name. + ctx.macros[node.Name] = node.Args + continue + } + if node.Snippet { + if ctx.nesting != 0 { + return res, ctx.Err("snippet declarations are only allowed at top-level") + } + if len(node.Args) != 0 { + return res, ctx.Err("snippet declarations can't have arguments") + } + + ctx.snippets[node.Name] = node.Children + continue + } + + if err := ctx.expandMacros(&node); err != nil { + return res, err + } + + res = append(res, node) + if shouldStop { + break + } + } + return res, nil +} + +func readTree(r io.Reader, location string, expansionDepth int) (nodes []Node, snips map[string][]Node, macros map[string][]string, err error) { + ctx := parseContext{ + Dispenser: lexer.NewDispenser(location, r), + snippets: make(map[string][]Node), + macros: map[string][]string{}, + nesting: -1, + fileLocation: location, + } + + root := Node{} + root.File = location + root.Line = 1 + // Before parsing starts the lexer's cursor points to the non-existent + // token before the first one. From readNodes viewpoint this is opening + // brace so we don't break any requirements here. + // + // For the same reason we use -1 as a starting nesting. So readNodes + // will see this as it is reading block at nesting level 0. + root.Children, err = ctx.readNodes() + if err != nil { + return root.Children, ctx.snippets, ctx.macros, err + } + + // There is no need to check ctx.nesting < 0 because it is checked by readNodes. + if ctx.nesting > 0 { + return root.Children, ctx.snippets, ctx.macros, ctx.Err("unexpected EOF when looking for }") + } + + root, err = ctx.expandImports(root, expansionDepth) + if err != nil { + return root.Children, ctx.snippets, ctx.macros, err + } + + return root.Children, ctx.snippets, ctx.macros, nil +} + +func Read(r io.Reader, location string) (nodes []Node, err error) { + nodes, _, _, err = readTree(r, location, 0) + nodes = expandEnvironment(nodes) + return +} diff --git a/framework/cfgparser/parse_test.go b/framework/cfgparser/parse_test.go new file mode 100644 index 0000000..32929b9 --- /dev/null +++ b/framework/cfgparser/parse_test.go @@ -0,0 +1,622 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package parser + +import ( + "os" + "reflect" + "strings" + "testing" +) + +var cases = []struct { + name string + cfg string + tree []Node + fail bool +}{ + { + "single directive without args", + `a`, + []Node{ + { + Name: "a", + Args: []string{}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "single directive with args", + `a a1 a2`, + []Node{ + { + Name: "a", + Args: []string{"a1", "a2"}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "single directive with empty braces", + `a { }`, + []Node{ + { + Name: "a", + Args: []string{}, + Children: []Node{}, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "single directive with arguments and empty braces", + `a a1 a2 { }`, + []Node{ + { + Name: "a", + Args: []string{"a1", "a2"}, + Children: []Node{}, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "single directive with a block", + `a a1 a2 { + a_child1 c1arg1 c1arg2 + a_child2 c2arg1 c2arg2 + }`, + []Node{ + { + Name: "a", + Args: []string{"a1", "a2"}, + Children: []Node{ + { + Name: "a_child1", + Args: []string{"c1arg1", "c1arg2"}, + Children: nil, + File: "test", + Line: 2, + }, + { + Name: "a_child2", + Args: []string{"c2arg1", "c2arg2"}, + Children: nil, + File: "test", + Line: 3, + }, + }, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "single directive with missing closing brace", + `a {`, + nil, + true, + }, + { + "single directive with missing opening brace", + `a }`, + nil, + true, + }, + { + "two directives", + `a + b`, + []Node{ + { + Name: "a", + Args: []string{}, + Children: nil, + File: "test", + Line: 1, + }, + { + Name: "b", + Args: []string{}, + Children: nil, + File: "test", + Line: 2, + }, + }, + false, + }, + { + "two directives with arguments", + `a a1 a2 + b b1 b2`, + []Node{ + { + Name: "a", + Args: []string{"a1", "a2"}, + Children: nil, + File: "test", + Line: 1, + }, + { + Name: "b", + Args: []string{"b1", "b2"}, + Children: nil, + File: "test", + Line: 2, + }, + }, + false, + }, + { + "backslash on the end of line", + `a a1 a2 \ + a3 a4`, + []Node{ + { + Name: "a", + Args: []string{"a1", "a2", "a3", "a4"}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "directive with missing closing brace on different line", + `a a1 a2 { + a_child1 c1arg1 c1arg2 + `, + nil, + true, + }, + { + "single directive with closing brace on children's line", + `a a1 a2 { + a_child1 c1arg1 c1arg2 + a_child2 c2arg1 c2arg2 } + b`, + []Node{ + { + Name: "a", + Args: []string{"a1", "a2"}, + Children: []Node{ + { + Name: "a_child1", + Args: []string{"c1arg1", "c1arg2"}, + Children: nil, + File: "test", + Line: 2, + }, + { + Name: "a_child2", + Args: []string{"c2arg1", "c2arg2"}, + Children: nil, + File: "test", + Line: 3, + }, + }, + File: "test", + Line: 1, + }, + { + Name: "b", + Args: []string{}, + Children: nil, + File: "test", + Line: 4, + }, + }, + false, + }, + { + "single directive with childrens on the same line", + `a a1 a2 { a_child1 c1arg1 c1arg2 }`, + []Node{ + { + Name: "a", + Args: []string{"a1", "a2"}, + Children: []Node{ + { + Name: "a_child1", + Args: []string{"c1arg1", "c1arg2"}, + Children: nil, + File: "test", + Line: 1, + }, + }, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "invalid directive name", + `a-a4@%8 whatever`, + nil, + true, + }, + { + "directive name starts with a digit", + `1w whatever`, + nil, + true, + }, + { + "missing block header", + `{ a_child1 c1arg1 c1arg2 }`, + nil, + true, + }, + { + "extra closing brace", + `a { + child1 + } } + `, + nil, + true, + }, + { + "extra opening brace", + `a { { + }`, + nil, + true, + }, + { + "closing brace in next block header", + `a { + } b b1`, + nil, + true, + }, + { + "environment variable expansion", + `a {env:TESTING_VARIABLE}`, + []Node{ + { + Name: "a", + Args: []string{"ABCDEF"}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "missing environment variable expansion (unix-like syntax)", + `a {env:TESTING_VARIABLE3}`, + []Node{ + { + Name: "a", + Args: []string{""}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "incomplete environment variable syntax", + `a {env:TESTING_VARIABLE`, + []Node{ + { + Name: "a", + Args: []string{"{env:TESTING_VARIABLE"}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "snippet expansion", + `(foo) { a } + import foo`, + []Node{ + { + Name: "a", + Args: []string{}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "snippet expansion inside a block", + `(foo) { a } + foo { + boo + import foo + }`, + []Node{ + { + Name: "foo", + Args: []string{}, + Children: []Node{ + { + Name: "boo", + Args: []string{}, + File: "test", + Line: 3, + }, + { + Name: "a", + Args: []string{}, + File: "test", + Line: 1, + }, + }, + File: "test", + Line: 2, + }, + }, + false, + }, + { + "missing snippet", + `import foo`, + nil, + true, + }, + { + "unlimited recursive snippet expansion", + `(foo) { import foo } + import foo`, + nil, + true, + }, + { + "snippet declaration with args", + `(foo) a b c { }`, + nil, + true, + }, + { + "snippet declaration inside block", + `abc { + (foo) { } + }`, + nil, + true, + }, + { + "block nesting limit", + `a ` + strings.Repeat("a { ", 1000) + strings.Repeat(" }", 1000), + nil, + true, + }, + { + "macro expansion, single argument", + `$(foo) = bar + dir $(foo)`, + []Node{ + { + Name: "dir", + Args: []string{"bar"}, + Children: nil, + File: "test", + Line: 2, + }, + }, + false, + }, + { + "macro expansion, inside argument", + `$(foo) = bar + dir aaa/$(foo)/bbb`, + []Node{ + { + Name: "dir", + Args: []string{"aaa/bar/bbb"}, + Children: nil, + File: "test", + Line: 2, + }, + }, + false, + }, + { + "macro expansion, inside argument, multi-value", + `$(foo) = bar baz + dir aaa/$(foo)/bbb`, + nil, + true, + }, + { + "macro expansion, multiple arguments", + `$(foo) = bar baz + dir $(foo)`, + []Node{ + { + Name: "dir", + Args: []string{"bar", "baz"}, + Children: nil, + File: "test", + Line: 2, + }, + }, + false, + }, + { + "macro expansion, undefined", + `dir $(foo)`, + []Node{ + { + Name: "dir", + Args: []string{}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "macro expansion, empty", + `$(foo) =`, + nil, + true, + }, + { + "macro expansion, name replacement", + `$(foo) = a b + $(foo) 1`, + nil, + true, + }, + { + "macro expansion, missing =", + `$(foo) a b + $(foo) 1`, + nil, + true, + }, + { + "macro expansion, not on top level", + `a { + $(foo) = a b + } + $(foo) 1`, + nil, + true, + }, + { + "macro expansion, nested", + `$(foo) = a + $(bar) = $(foo) b + dir $(bar)`, + []Node{ + { + Name: "dir", + Args: []string{"a", "b"}, + Children: nil, + File: "test", + Line: 3, + }, + }, + false, + }, + { + "macro expansion, used inside snippet", + `$(foo) = a + (bar) { + dir $(foo) + } + import bar`, + []Node{ + { + Name: "dir", + Args: []string{"a"}, + Children: nil, + File: "test", + Line: 3, + }, + }, + false, + }, + { + "macro expansion, used inside snippet, defined after", + ` + (bar) { + dir $(foo) + } + $(foo) = a + import bar`, + []Node{ + { + Name: "dir", + Args: []string{}, + Children: nil, + File: "test", + Line: 3, + }, + }, + false, + }, +} + +func printTree(t *testing.T, root Node, indent int) { + t.Log(strings.Repeat(" ", indent)+root.Name, root.Args) + for _, child := range root.Children { + t.Log(child, indent+1) + } +} + +func TestRead(t *testing.T) { + os.Setenv("TESTING_VARIABLE", "ABCDEF") + os.Setenv("TESTING_VARIABLE2", "ABC2 DEF2") + + for _, case_ := range cases { + t.Run(case_.name, func(t *testing.T) { + tree, err := Read(strings.NewReader(case_.cfg), "test") + if !case_.fail && err != nil { + t.Error("unexpected failure:", err) + return + } + if case_.fail { + if err == nil { + t.Log("expected failure but Read succeeded") + t.Log("got tree:") + t.Logf("%+v", tree) + for _, node := range tree { + printTree(t, node, 0) + } + t.Fail() + return + } + return + } + + if !reflect.DeepEqual(case_.tree, tree) { + t.Log("parse result mismatch") + t.Log("expected:") + t.Logf("%+#v", case_.tree) + for _, node := range case_.tree { + printTree(t, node, 0) + } + t.Log("actual:") + t.Logf("%+#v", tree) + for _, node := range tree { + printTree(t, node, 0) + } + t.Fail() + } + }) + } +} diff --git a/framework/config/config.go b/framework/config/config.go new file mode 100644 index 0000000..3dd9911 --- /dev/null +++ b/framework/config/config.go @@ -0,0 +1,36 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package config + +import ( + "fmt" + + parser "github.com/foxcpp/maddy/framework/cfgparser" +) + +type ( + Node = parser.Node +) + +func NodeErr(node Node, f string, args ...interface{}) error { + if node.File == "" { + return fmt.Errorf(f, args...) + } + return fmt.Errorf("%s:%d: %s", node.File, node.Line, fmt.Sprintf(f, args...)) +} diff --git a/framework/config/directories.go b/framework/config/directories.go new file mode 100644 index 0000000..013dfc9 --- /dev/null +++ b/framework/config/directories.go @@ -0,0 +1,47 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package config + +var ( + // StateDirectory contains the path to the directory that + // should be used to store any data that should be + // preserved between sessions. + // + // Value of this variable must not change after initialization + // in cmd/maddy/main.go. + StateDirectory string + + // RuntimeDirectory contains the path to the directory that + // should be used to store any temporary data. + // + // It should be preferred over os.TempDir, which is + // global and world-readable on most systems, while + // RuntimeDirectory can be dedicated for maddy. + // + // Value of this variable must not change after initialization + // in cmd/maddy/main.go. + RuntimeDirectory string + + // LibexecDirectory contains the path to the directory + // where helper binaries should be searched. + // + // Value of this variable must not change after initialization + // in cmd/maddy/main.go. + LibexecDirectory string +) diff --git a/framework/config/endpoint.go b/framework/config/endpoint.go new file mode 100644 index 0000000..210d57b --- /dev/null +++ b/framework/config/endpoint.go @@ -0,0 +1,142 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package config + +import ( + "fmt" + "net" + "net/url" + "path/filepath" + "strings" +) + +// Endpoint represents a site address. It contains the original input value, +// and the component parts of an address. The component parts may be updated to +// the correct values as setup proceeds, but the original value should never be +// changed. +type Endpoint struct { + Original, Scheme, Host, Port, Path string +} + +// String returns a human-friendly print of the address. +func (e Endpoint) String() string { + if e.Original != "" { + return e.Original + } + + if e.Scheme == "unix" { + return "unix://" + e.Path + } + + if e.Host == "" && e.Port == "" { + return "" + } + s := e.Scheme + if s != "" { + s += "://" + } + + host := e.Host + if strings.Contains(host, ":") { + host = "[" + host + "]" + } + s += host + + if e.Port != "" { + s += ":" + e.Port + } + if e.Path != "" { + s += e.Path + } + return s +} + +func (e Endpoint) Network() string { + if e.Scheme == "unix" { + return "unix" + } + return "tcp" +} + +func (e Endpoint) Address() string { + if e.Scheme == "unix" { + return e.Path + } + return net.JoinHostPort(e.Host, e.Port) +} + +func (e Endpoint) IsTLS() bool { + return e.Scheme == "tls" +} + +// ParseEndpoint parses an endpoint string into a structured format with separate +// scheme, host, port, and path portions, as well as the original input string. +func ParseEndpoint(str string) (Endpoint, error) { + input := str + + u, err := url.Parse(str) + if err != nil { + return Endpoint{}, err + } + + switch u.Scheme { + case "tcp", "tls": + // ALL GREEN + + // scheme:OPAQUE URL syntax + if u.Host == "" && u.Opaque != "" { + u.Host = u.Opaque + } + case "unix": + // scheme:OPAQUE URL syntax + if u.Path == "" && u.Opaque != "" { + u.Path = u.Opaque + } + + var actualPath string + if u.Host != "" { + actualPath += u.Host + } + if u.Path != "" { + actualPath += u.Path + } + + if !filepath.IsAbs(actualPath) { + actualPath = filepath.Join(RuntimeDirectory, actualPath) + } + + return Endpoint{Original: input, Scheme: u.Scheme, Path: actualPath}, err + default: + return Endpoint{}, fmt.Errorf("unsupported scheme: %s (%+v)", input, u) + } + + // separate host and port + host, port, err := net.SplitHostPort(u.Host) + if err != nil { + host, port, err = net.SplitHostPort(u.Host + ":") + if err != nil { + host = u.Host + } + } + if port == "" { + return Endpoint{}, fmt.Errorf("port is required") + } + + return Endpoint{Original: input, Scheme: u.Scheme, Host: host, Port: port, Path: u.Path}, err +} diff --git a/framework/config/endpoint_test.go b/framework/config/endpoint_test.go new file mode 100644 index 0000000..b786ef0 --- /dev/null +++ b/framework/config/endpoint_test.go @@ -0,0 +1,55 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package config + +import ( + "reflect" + "testing" +) + +func TestStandardizeAddress(t *testing.T) { + for _, expected := range []Endpoint{ + {Original: "tcp://0.0.0.0:10025", Scheme: "tcp", Host: "0.0.0.0", Port: "10025"}, + {Original: "tcp://[::]:10025", Scheme: "tcp", Host: "::", Port: "10025"}, + {Original: "tcp:127.0.0.1:10025", Scheme: "tcp", Host: "127.0.0.1", Port: "10025"}, + {Original: "unix://path", Scheme: "unix", Host: "", Path: "path", Port: ""}, + {Original: "unix:path", Scheme: "unix", Host: "", Path: "path", Port: ""}, + {Original: "unix:/path", Scheme: "unix", Host: "", Path: "/path", Port: ""}, + {Original: "unix:///path", Scheme: "unix", Host: "", Path: "/path", Port: ""}, + {Original: "unix://also/path", Scheme: "unix", Host: "", Path: "also/path", Port: ""}, + {Original: "unix:///also/path", Scheme: "unix", Host: "", Path: "/also/path", Port: ""}, + {Original: "tls://0.0.0.0:10025", Scheme: "tls", Host: "0.0.0.0", Port: "10025"}, + {Original: "tls:0.0.0.0:10025", Scheme: "tls", Host: "0.0.0.0", Port: "10025"}, + } { + actual, err := ParseEndpoint(expected.Original) + if err != nil { + t.Errorf("Unexpected failure for %s: %v", expected.Original, err) + return + } + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Didn't parse URL %q correctly\ngot %#v\nwant %#v", expected.Original, actual, expected) + continue + } + + if actual.String() != expected.Original { + t.Errorf("actual.String() = %s, want %s", actual.String(), expected.Original) + } + } +} diff --git a/framework/config/lexer/LICENSE.APACHE b/framework/config/lexer/LICENSE.APACHE new file mode 100644 index 0000000..8dada3e --- /dev/null +++ b/framework/config/lexer/LICENSE.APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/framework/config/lexer/README.md b/framework/config/lexer/README.md new file mode 100644 index 0000000..554a7a2 --- /dev/null +++ b/framework/config/lexer/README.md @@ -0,0 +1,20 @@ +caddyfile lexer copied from [caddy](https://github.com/caddyserver/caddy) project. + +Taken from the following commit: +``` +commit ed4c2775e46b924d4851e04cc281633b1b2c15af +Author: Alexander Danilov +Date: Wed Aug 21 20:13:34 2019 +0300 + + main: log caddy version on start (#2717) + +``` + +License of the original code is included in LICENSE.APACHE file in this +directory. + +No signficant changes was made to the code (e.g. it is safe to update it from +caddy repo). + +The code is copied because caddy brings quite a lot of dependencies we don't +use and this slows down many tools. diff --git a/framework/config/lexer/dispenser.go b/framework/config/lexer/dispenser.go new file mode 100644 index 0000000..db710f1 --- /dev/null +++ b/framework/config/lexer/dispenser.go @@ -0,0 +1,264 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Copyright 2015 Light Code Labs, LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lexer + +import ( + "errors" + "fmt" + "io" + "strings" +) + +// Dispenser is a type that dispenses tokens, similarly to a lexer, +// except that it can do so with some notion of structure and has +// some really convenient methods. +type Dispenser struct { + filename string + tokens []Token + cursor int + nesting int +} + +// NewDispenser returns a Dispenser, ready to use for parsing the given input. +func NewDispenser(filename string, input io.Reader) Dispenser { + tokens, _ := allTokens(input) // ignoring error because nothing to do with it + return Dispenser{ + filename: filename, + tokens: tokens, + cursor: -1, + } +} + +// NewDispenserTokens returns a Dispenser filled with the given tokens. +func NewDispenserTokens(filename string, tokens []Token) Dispenser { + return Dispenser{ + filename: filename, + tokens: tokens, + cursor: -1, + } +} + +// Next loads the next token. Returns true if a token +// was loaded; false otherwise. If false, all tokens +// have been consumed. +func (d *Dispenser) Next() bool { + if d.cursor < len(d.tokens)-1 { + d.cursor++ + return true + } + return false +} + +// NextArg loads the next token if it is on the same +// line. Returns true if a token was loaded; false +// otherwise. If false, all tokens on the line have +// been consumed. It handles imported tokens correctly. +func (d *Dispenser) NextArg() bool { + if d.cursor < 0 { + d.cursor++ + return true + } + if d.cursor >= len(d.tokens) { + return false + } + if d.cursor < len(d.tokens)-1 && + d.tokens[d.cursor].File == d.tokens[d.cursor+1].File && + d.tokens[d.cursor].Line+d.numLineBreaks(d.cursor) == d.tokens[d.cursor+1].Line { + d.cursor++ + return true + } + return false +} + +// NextLine loads the next token only if it is not on the same +// line as the current token, and returns true if a token was +// loaded; false otherwise. If false, there is not another token +// or it is on the same line. It handles imported tokens correctly. +func (d *Dispenser) NextLine() bool { + if d.cursor < 0 { + d.cursor++ + return true + } + if d.cursor >= len(d.tokens) { + return false + } + if d.cursor < len(d.tokens)-1 && + (d.tokens[d.cursor].File != d.tokens[d.cursor+1].File || + d.tokens[d.cursor].Line+d.numLineBreaks(d.cursor) < d.tokens[d.cursor+1].Line) { + d.cursor++ + return true + } + return false +} + +// NextBlock can be used as the condition of a for loop +// to load the next token as long as it opens a block or +// is already in a block. It returns true if a token was +// loaded, or false when the block's closing curly brace +// was loaded and thus the block ended. Nested blocks are +// not supported. +func (d *Dispenser) NextBlock() bool { + if d.nesting > 0 { + d.Next() + if d.Val() == "}" { + d.nesting-- + return false + } + return true + } + if !d.NextArg() { // block must open on same line + return false + } + if d.Val() != "{" { + d.cursor-- // roll back if not opening brace + return false + } + d.Next() + if d.Val() == "}" { + // Open and then closed right away + return false + } + d.nesting++ + return true +} + +// Val gets the text of the current token. If there is no token +// loaded, it returns empty string. +func (d *Dispenser) Val() string { + if d.cursor < 0 || d.cursor >= len(d.tokens) { + return "" + } + return d.tokens[d.cursor].Text +} + +// Line gets the line number of the current token. If there is no token +// loaded, it returns 0. +func (d *Dispenser) Line() int { + if d.cursor < 0 || d.cursor >= len(d.tokens) { + return 0 + } + return d.tokens[d.cursor].Line +} + +// File gets the filename of the current token. If there is no token loaded, +// it returns the filename originally given when parsing started. +func (d *Dispenser) File() string { + if d.cursor < 0 || d.cursor >= len(d.tokens) { + return d.filename + } + if tokenFilename := d.tokens[d.cursor].File; tokenFilename != "" { + return tokenFilename + } + return d.filename +} + +// Args is a convenience function that loads the next arguments +// (tokens on the same line) into an arbitrary number of strings +// pointed to in targets. If there are fewer tokens available +// than string pointers, the remaining strings will not be changed +// and false will be returned. If there were enough tokens available +// to fill the arguments, then true will be returned. +func (d *Dispenser) Args(targets ...*string) bool { + enough := true + for i := 0; i < len(targets); i++ { + if !d.NextArg() { + enough = false + break + } + *targets[i] = d.Val() + } + return enough +} + +// RemainingArgs loads any more arguments (tokens on the same line) +// into a slice and returns them. Open curly brace tokens also indicate +// the end of arguments, and the curly brace is not included in +// the return value nor is it loaded. +func (d *Dispenser) RemainingArgs() []string { + var args []string + + for d.NextArg() { + if d.Val() == "{" { + d.cursor-- + break + } + args = append(args, d.Val()) + } + + return args +} + +// ArgErr returns an argument error, meaning that another +// argument was expected but not found. In other words, +// a line break or open curly brace was encountered instead of +// an argument. +func (d *Dispenser) ArgErr() error { + if d.Val() == "{" { + return d.Err("Unexpected token '{', expecting argument") + } + return d.Errf("Wrong argument count or unexpected line ending after '%s'", d.Val()) +} + +// SyntaxErr creates a generic syntax error which explains what was +// found and what was expected. +func (d *Dispenser) SyntaxErr(expected string) error { + msg := fmt.Sprintf("%s:%d - Syntax error: Unexpected token '%s', expecting '%s'", d.File(), d.Line(), d.Val(), expected) + return errors.New(msg) +} + +// EOFErr returns an error indicating that the dispenser reached +// the end of the input when searching for the next token. +func (d *Dispenser) EOFErr() error { + return d.Errf("Unexpected EOF") +} + +// Err generates a custom parse-time error with a message of msg. +func (d *Dispenser) Err(msg string) error { + msg = fmt.Sprintf("%s:%d - Error during parsing: %s", d.File(), d.Line(), msg) + return errors.New(msg) +} + +// Errf is like Err, but for formatted error messages +func (d *Dispenser) Errf(format string, args ...interface{}) error { + return d.Err(fmt.Sprintf(format, args...)) +} + +// numLineBreaks counts how many line breaks are in the token +// value given by the token index tknIdx. It returns 0 if the +// token does not exist or there are no line breaks. +func (d *Dispenser) numLineBreaks(tknIdx int) int { + if tknIdx < 0 || tknIdx >= len(d.tokens) { + return 0 + } + return strings.Count(d.tokens[tknIdx].Text, "\n") +} diff --git a/framework/config/lexer/dispenser_test.go b/framework/config/lexer/dispenser_test.go new file mode 100644 index 0000000..1f75bb1 --- /dev/null +++ b/framework/config/lexer/dispenser_test.go @@ -0,0 +1,324 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Copyright 2015 Light Code Labs, LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lexer + +import ( + "reflect" + "strings" + "testing" +) + +func TestDispenser_Val_Next(t *testing.T) { + input := `host:port + dir1 arg1 + dir2 arg2 arg3 + dir3` + d := NewDispenser("Testfile", strings.NewReader(input)) + + if val := d.Val(); val != "" { + t.Fatalf("Val(): Should return empty string when no token loaded; got '%s'", val) + } + + assertNext := func(shouldLoad bool, expectedCursor int, expectedVal string) { + if loaded := d.Next(); loaded != shouldLoad { + t.Errorf("Next(): Expected %v but got %v instead (val '%s')", shouldLoad, loaded, d.Val()) + } + if d.cursor != expectedCursor { + t.Errorf("Expected cursor to be %d, but was %d", expectedCursor, d.cursor) + } + if d.nesting != 0 { + t.Errorf("Nesting should be 0, was %d instead", d.nesting) + } + if val := d.Val(); val != expectedVal { + t.Errorf("Val(): Expected '%s' but got '%s'", expectedVal, val) + } + } + + assertNext(true, 0, "host:port") + assertNext(true, 1, "dir1") + assertNext(true, 2, "arg1") + assertNext(true, 3, "dir2") + assertNext(true, 4, "arg2") + assertNext(true, 5, "arg3") + assertNext(true, 6, "dir3") + // Note: This next test simply asserts existing behavior. + // If desired, we may wish to empty the token value after + // reading past the EOF. Open an issue if you want this change. + assertNext(false, 6, "dir3") +} + +func TestDispenser_NextArg(t *testing.T) { + input := `dir1 arg1 + dir2 arg2 arg3 + dir3` + d := NewDispenser("Testfile", strings.NewReader(input)) + + assertNext := func(shouldLoad bool, expectedVal string, expectedCursor int) { + if d.Next() != shouldLoad { + t.Errorf("Next(): Should load token but got false instead (val: '%s')", d.Val()) + } + if d.cursor != expectedCursor { + t.Errorf("Next(): Expected cursor to be at %d, but it was %d", expectedCursor, d.cursor) + } + if val := d.Val(); val != expectedVal { + t.Errorf("Val(): Expected '%s' but got '%s'", expectedVal, val) + } + } + + assertNextArg := func(expectedVal string, loadAnother bool, expectedCursor int) { + if !d.NextArg() { + t.Error("NextArg(): Should load next argument but got false instead") + } + if d.cursor != expectedCursor { + t.Errorf("NextArg(): Expected cursor to be at %d, but it was %d", expectedCursor, d.cursor) + } + if val := d.Val(); val != expectedVal { + t.Errorf("Val(): Expected '%s' but got '%s'", expectedVal, val) + } + if !loadAnother { + if d.NextArg() { + t.Fatalf("NextArg(): Should NOT load another argument, but got true instead (val: '%s')", d.Val()) + } + if d.cursor != expectedCursor { + t.Errorf("NextArg(): Expected cursor to remain at %d, but it was %d", expectedCursor, d.cursor) + } + } + } + + assertNext(true, "dir1", 0) + assertNextArg("arg1", false, 1) + assertNext(true, "dir2", 2) + assertNextArg("arg2", true, 3) + assertNextArg("arg3", false, 4) + assertNext(true, "dir3", 5) + assertNext(false, "dir3", 5) +} + +func TestDispenser_NextLine(t *testing.T) { + input := `host:port + dir1 arg1 + dir2 arg2 arg3` + d := NewDispenser("Testfile", strings.NewReader(input)) + + assertNextLine := func(shouldLoad bool, expectedVal string, expectedCursor int) { + if d.NextLine() != shouldLoad { + t.Errorf("NextLine(): Should load token but got false instead (val: '%s')", d.Val()) + } + if d.cursor != expectedCursor { + t.Errorf("NextLine(): Expected cursor to be %d, instead was %d", expectedCursor, d.cursor) + } + if val := d.Val(); val != expectedVal { + t.Errorf("Val(): Expected '%s' but got '%s'", expectedVal, val) + } + } + + assertNextLine(true, "host:port", 0) + assertNextLine(true, "dir1", 1) + assertNextLine(false, "dir1", 1) + d.Next() // arg1 + assertNextLine(true, "dir2", 3) + assertNextLine(false, "dir2", 3) + d.Next() // arg2 + assertNextLine(false, "arg2", 4) + d.Next() // arg3 + assertNextLine(false, "arg3", 5) +} + +func TestDispenser_NextBlock(t *testing.T) { + input := `foobar1 { + sub1 arg1 + sub2 + } + foobar2 { + }` + d := NewDispenser("Testfile", strings.NewReader(input)) + + assertNextBlock := func(shouldLoad bool, expectedCursor, expectedNesting int) { + if loaded := d.NextBlock(); loaded != shouldLoad { + t.Errorf("NextBlock(): Should return %v but got %v", shouldLoad, loaded) + } + if d.cursor != expectedCursor { + t.Errorf("NextBlock(): Expected cursor to be %d, was %d", expectedCursor, d.cursor) + } + if d.nesting != expectedNesting { + t.Errorf("NextBlock(): Nesting should be %d, not %d", expectedNesting, d.nesting) + } + } + + assertNextBlock(false, -1, 0) + d.Next() // foobar1 + assertNextBlock(true, 2, 1) + assertNextBlock(true, 3, 1) + assertNextBlock(true, 4, 1) + assertNextBlock(false, 5, 0) + d.Next() // foobar2 + assertNextBlock(false, 8, 0) // empty block is as if it didn't exist +} + +func TestDispenser_Args(t *testing.T) { + var s1, s2, s3 string + input := `dir1 arg1 arg2 arg3 + dir2 arg4 arg5 + dir3 arg6 arg7 + dir4` + d := NewDispenser("Testfile", strings.NewReader(input)) + + d.Next() // dir1 + + // As many strings as arguments + if all := d.Args(&s1, &s2, &s3); !all { + t.Error("Args(): Expected true, got false") + } + if s1 != "arg1" { + t.Errorf("Args(): Expected s1 to be 'arg1', got '%s'", s1) + } + if s2 != "arg2" { + t.Errorf("Args(): Expected s2 to be 'arg2', got '%s'", s2) + } + if s3 != "arg3" { + t.Errorf("Args(): Expected s3 to be 'arg3', got '%s'", s3) + } + + d.Next() // dir2 + + // More strings than arguments + if all := d.Args(&s1, &s2, &s3); all { + t.Error("Args(): Expected false, got true") + } + if s1 != "arg4" { + t.Errorf("Args(): Expected s1 to be 'arg4', got '%s'", s1) + } + if s2 != "arg5" { + t.Errorf("Args(): Expected s2 to be 'arg5', got '%s'", s2) + } + if s3 != "arg3" { + t.Errorf("Args(): Expected s3 to be unchanged ('arg3'), instead got '%s'", s3) + } + + // (quick cursor check just for kicks and giggles) + if d.cursor != 6 { + t.Errorf("Cursor should be 6, but is %d", d.cursor) + } + + d.Next() // dir3 + + // More arguments than strings + if all := d.Args(&s1); !all { + t.Error("Args(): Expected true, got false") + } + if s1 != "arg6" { + t.Errorf("Args(): Expected s1 to be 'arg6', got '%s'", s1) + } + + d.Next() // dir4 + + // No arguments or strings + if all := d.Args(); !all { + t.Error("Args(): Expected true, got false") + } + + // No arguments but at least one string + if all := d.Args(&s1); all { + t.Error("Args(): Expected false, got true") + } +} + +func TestDispenser_RemainingArgs(t *testing.T) { + input := `dir1 arg1 arg2 arg3 + dir2 arg4 arg5 + dir3 arg6 { arg7 + dir4` + d := NewDispenser("Testfile", strings.NewReader(input)) + + d.Next() // dir1 + + args := d.RemainingArgs() + if expected := []string{"arg1", "arg2", "arg3"}; !reflect.DeepEqual(args, expected) { + t.Errorf("RemainingArgs(): Expected %v, got %v", expected, args) + } + + d.Next() // dir2 + + args = d.RemainingArgs() + if expected := []string{"arg4", "arg5"}; !reflect.DeepEqual(args, expected) { + t.Errorf("RemainingArgs(): Expected %v, got %v", expected, args) + } + + d.Next() // dir3 + + args = d.RemainingArgs() + if expected := []string{"arg6"}; !reflect.DeepEqual(args, expected) { + t.Errorf("RemainingArgs(): Expected %v, got %v", expected, args) + } + + d.Next() // { + d.Next() // arg7 + d.Next() // dir4 + + args = d.RemainingArgs() + if len(args) != 0 { + t.Errorf("RemainingArgs(): Expected %v, got %v", []string{}, args) + } +} + +func TestDispenser_ArgErr_Err(t *testing.T) { + input := `dir1 { + } + dir2 arg1 arg2` + d := NewDispenser("Testfile", strings.NewReader(input)) + + d.cursor = 1 // { + + if err := d.ArgErr(); err == nil || !strings.Contains(err.Error(), "{") { + t.Errorf("ArgErr(): Expected an error message with { in it, but got '%v'", err) + } + + d.cursor = 5 // arg2 + + if err := d.ArgErr(); err == nil || !strings.Contains(err.Error(), "arg2") { + t.Errorf("ArgErr(): Expected an error message with 'arg2' in it; got '%v'", err) + } + + err := d.Err("foobar") + if err == nil { + t.Fatalf("Err(): Expected an error, got nil") + } + + if !strings.Contains(err.Error(), "Testfile:3") { + t.Errorf("Expected error message with filename:line in it; got '%v'", err) + } + + if !strings.Contains(err.Error(), "foobar") { + t.Errorf("Expected error message with custom message in it ('foobar'); got '%v'", err) + } +} diff --git a/framework/config/lexer/lexer.go b/framework/config/lexer/lexer.go new file mode 100644 index 0000000..38952df --- /dev/null +++ b/framework/config/lexer/lexer.go @@ -0,0 +1,178 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Copyright 2015 Light Code Labs, LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lexer + +import ( + "bufio" + "io" + "unicode" +) + +type ( + // lexer is a utility which can get values, token by + // token, from a Reader. A token is a word, and tokens + // are separated by whitespace. A word can be enclosed + // in quotes if it contains whitespace. + lexer struct { + reader *bufio.Reader + token Token + line int + + lastErr error + } + + // Token represents a single parsable unit. + Token struct { + File string + Line int + Text string + } +) + +// load prepares the lexer to scan an input for tokens. +// It discards any leading byte order mark. +func (l *lexer) load(input io.Reader) error { + l.reader = bufio.NewReader(input) + l.line = 1 + + // discard byte order mark, if present + firstCh, _, err := l.reader.ReadRune() + if err != nil { + return err + } + if firstCh != 0xFEFF { + err := l.reader.UnreadRune() + if err != nil { + return err + } + } + + return nil +} + +func (l *lexer) err() error { + return l.lastErr +} + +// next loads the next token into the lexer. +// +// A token is delimited by whitespace, unless the token starts with a quotes +// character (") in which case the token goes until the closing quotes (the +// enclosing quotes are not included). Inside quoted strings, quotes may be +// escaped with a preceding \ character. No other chars may be escaped. Curly +// braces ('{', '}') are emitted as a separate tokens. +// +// The rest of the line is skipped if a "#" character is read in. +// +// Returns true if a token was loaded; false otherwise. If read from +// underlying Reader fails, next() returns false and err() will return the +// error occurred. +func (l *lexer) next() bool { + var val []rune + var comment, quoted, escaped bool + + makeToken := func() bool { + l.token.Text = string(val) + l.lastErr = nil + return true + } + + for { + ch, _, err := l.reader.ReadRune() + if err != nil { + if len(val) > 0 { + return makeToken() + } + if err == io.EOF { + return false + } + l.lastErr = err + return false + } + + if quoted { + if !escaped { + if ch == '\\' { + escaped = true + continue + } else if ch == '"' { + return makeToken() + } + } + if ch == '\n' { + l.line++ + } + if escaped { + // only escape quotes + if ch != '"' { + val = append(val, '\\') + } + } + val = append(val, ch) + escaped = false + continue + } + + if unicode.IsSpace(ch) { + if ch == '\r' { + continue + } + if ch == '\n' { + l.line++ + comment = false + } + if len(val) > 0 { + return makeToken() + } + continue + } + + if ch == '#' { + comment = true + } + + if comment { + continue + } + + if len(val) == 0 { + l.token = Token{Line: l.line} + if ch == '"' { + quoted = true + continue + } + } + + val = append(val, ch) + } +} diff --git a/framework/config/lexer/lexer_test.go b/framework/config/lexer/lexer_test.go new file mode 100644 index 0000000..5f42f2f --- /dev/null +++ b/framework/config/lexer/lexer_test.go @@ -0,0 +1,206 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Copyright 2015 Light Code Labs, LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lexer + +import ( + "log" + "strings" + "testing" +) + +type lexerTestCase struct { + input string + expected []Token +} + +func TestLexer(t *testing.T) { + testCases := []lexerTestCase{ + { + input: `host:123`, + expected: []Token{ + {Line: 1, Text: "host:123"}, + }, + }, + { + input: `host:123 + + directive`, + expected: []Token{ + {Line: 1, Text: "host:123"}, + {Line: 3, Text: "directive"}, + }, + }, + { + input: `host:123 { + directive + }`, + expected: []Token{ + {Line: 1, Text: "host:123"}, + {Line: 1, Text: "{"}, + {Line: 2, Text: "directive"}, + {Line: 3, Text: "}"}, + }, + }, + { + input: `host:123 { directive }`, + expected: []Token{ + {Line: 1, Text: "host:123"}, + {Line: 1, Text: "{"}, + {Line: 1, Text: "directive"}, + {Line: 1, Text: "}"}, + }, + }, + { + input: `host:123 { + #comment + directive + # comment + foobar # another comment + }`, + expected: []Token{ + {Line: 1, Text: "host:123"}, + {Line: 1, Text: "{"}, + {Line: 3, Text: "directive"}, + {Line: 5, Text: "foobar"}, + {Line: 6, Text: "}"}, + }, + }, + { + input: `a "quoted value" b + foobar`, + expected: []Token{ + {Line: 1, Text: "a"}, + {Line: 1, Text: "quoted value"}, + {Line: 1, Text: "b"}, + {Line: 2, Text: "foobar"}, + }, + }, + { + input: `A "quoted \"value\" inside" B`, + expected: []Token{ + {Line: 1, Text: "A"}, + {Line: 1, Text: `quoted "value" inside`}, + {Line: 1, Text: "B"}, + }, + }, + { + input: `"don't\escape"`, + expected: []Token{ + {Line: 1, Text: `don't\escape`}, + }, + }, + { + input: `"don't\\escape"`, + expected: []Token{ + {Line: 1, Text: `don't\\escape`}, + }, + }, + { + input: `A "quoted value with line + break inside" { + foobar + }`, + expected: []Token{ + {Line: 1, Text: "A"}, + {Line: 1, Text: "quoted value with line\n\t\t\t\t\tbreak inside"}, + {Line: 2, Text: "{"}, + {Line: 3, Text: "foobar"}, + {Line: 4, Text: "}"}, + }, + }, + { + input: `"C:\php\php-cgi.exe"`, + expected: []Token{ + {Line: 1, Text: `C:\php\php-cgi.exe`}, + }, + }, + { + input: `empty "" string`, + expected: []Token{ + {Line: 1, Text: `empty`}, + {Line: 1, Text: ``}, + {Line: 1, Text: `string`}, + }, + }, + { + input: "skip those\r\nCR characters", + expected: []Token{ + {Line: 1, Text: "skip"}, + {Line: 1, Text: "those"}, + {Line: 2, Text: "CR"}, + {Line: 2, Text: "characters"}, + }, + }, + { + input: "\xEF\xBB\xBF:8080", // test with leading byte order mark + expected: []Token{ + {Line: 1, Text: ":8080"}, + }, + }, + } + + for i, testCase := range testCases { + actual := tokenize(testCase.input) + lexerCompare(t, i, testCase.expected, actual) + } +} + +func tokenize(input string) (tokens []Token) { + l := lexer{} + if err := l.load(strings.NewReader(input)); err != nil { + log.Printf("[ERROR] load failed: %v", err) + } + for l.next() { + tokens = append(tokens, l.token) + } + return +} + +func lexerCompare(t *testing.T, n int, expected, actual []Token) { + if len(expected) != len(actual) { + t.Errorf("Test case %d: expected %d token(s) but got %d", n, len(expected), len(actual)) + } + + for i := 0; i < len(actual) && i < len(expected); i++ { + if actual[i].Line != expected[i].Line { + t.Errorf("Test case %d token %d ('%s'): expected line %d but was line %d", + n, i, expected[i].Text, expected[i].Line, actual[i].Line) + break + } + if actual[i].Text != expected[i].Text { + t.Errorf("Test case %d token %d: expected text '%s' but was '%s'", + n, i, expected[i].Text, actual[i].Text) + break + } + } +} diff --git a/framework/config/lexer/parse.go b/framework/config/lexer/parse.go new file mode 100644 index 0000000..34c8d79 --- /dev/null +++ b/framework/config/lexer/parse.go @@ -0,0 +1,42 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package lexer + +import ( + "io" +) + +// allTokens lexes the entire input, but does not parse it. +// It returns all the tokens from the input, unstructured +// and in order. +func allTokens(input io.Reader) ([]Token, error) { + l := new(lexer) + err := l.load(input) + if err != nil { + return nil, err + } + var tokens []Token + for l.next() { + tokens = append(tokens, l.token) + } + if err := l.err(); err != nil { + return nil, err + } + return tokens, nil +} diff --git a/framework/config/map.go b/framework/config/map.go new file mode 100644 index 0000000..c85c19f --- /dev/null +++ b/framework/config/map.go @@ -0,0 +1,743 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package config + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "time" + "unicode" +) + +type matcher struct { + name string + required bool + inheritGlobal bool + defaultVal func() (interface{}, error) + mapper func(*Map, Node) (interface{}, error) + store *reflect.Value + + customCallback func(*Map, Node) error +} + +func (m *matcher) assign(val interface{}) { + valRefl := reflect.ValueOf(val) + // Convert untyped nil into typed nil. Otherwise it will panic. + if !valRefl.IsValid() { + valRefl = reflect.Zero(m.store.Type()) + } + + m.store.Set(valRefl) +} + +// Map structure implements reflection-based conversion between configuration +// directives and Go variables. +type Map struct { + allowUnknown bool + + // All values saved by Map during processing. + Values map[string]interface{} + + entries map[string]matcher + + // Values used by Process as default values if inheritGlobal is true. + Globals map[string]interface{} + // Config block used by Process. + Block Node +} + +func NewMap(globals map[string]interface{}, block Node) *Map { + return &Map{Globals: globals, Block: block} +} + +// AllowUnknown makes config.Map skip unknown configuration directives instead +// of failing. +func (m *Map) AllowUnknown() { + m.allowUnknown = true +} + +// EnumList maps a configuration directive to a []string variable. +// +// Directive must be in form 'name string1 string2' where each string should be from *allowed* +// slice. At least one argument should be present. +// +// See Map.Custom for description of inheritGlobal and required. +func (m *Map) EnumList(name string, inheritGlobal, required bool, allowed, defaultVal []string, store *[]string) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare a block here") + } + if len(node.Args) == 0 { + return nil, NodeErr(node, "expected at least one argument") + } + + for _, arg := range node.Args { + isAllowed := false + for _, str := range allowed { + if str == arg { + isAllowed = true + } + } + if !isAllowed { + return nil, NodeErr(node, "invalid argument, valid values are: %v", allowed) + } + } + + return node.Args, nil + }, store) +} + +// Enum maps a configuration directive to a string variable. +// +// Directive must be in form 'name string' where string should be from *allowed* +// slice. That string argument will be stored in store variable. +// +// See Map.Custom for description of inheritGlobal and required. +func (m *Map) Enum(name string, inheritGlobal, required bool, allowed []string, defaultVal string, store *string) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare a block here") + } + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected exactly one argument") + } + + for _, str := range allowed { + if str == node.Args[0] { + return node.Args[0], nil + } + } + + return nil, NodeErr(node, "invalid argument, valid values are: %v", allowed) + }, store) +} + +// EnumMapped is similar to Map.Enum but maps a stirng to a custom type. +func EnumMapped[V any](m *Map, name string, inheritGlobal, required bool, mapped map[string]V, defaultVal V, store *V) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare a block here") + } + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected exactly one argument") + } + + val, ok := mapped[node.Args[0]] + if !ok { + validValues := make([]string, 0, len(mapped)) + for k := range mapped { + validValues = append(validValues, k) + } + return nil, NodeErr(node, "invalid argument, valid values are: %v", validValues) + } + + return val, nil + }, store) +} + +// EnumListMapped is similar to Map.EnumList but maps a stirng to a custom type. +func EnumListMapped[V any](m *Map, name string, inheritGlobal, required bool, mapped map[string]V, defaultVal []V, store *[]V) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare a block here") + } + if len(node.Args) == 0 { + return nil, NodeErr(node, "expected at least one argument") + } + + values := make([]V, 0, len(node.Args)) + for _, arg := range node.Args { + val, ok := mapped[arg] + if !ok { + validValues := make([]string, 0, len(mapped)) + for k := range mapped { + validValues = append(validValues, k) + } + return nil, NodeErr(node, "invalid argument, valid values are: %v", validValues) + } + values = append(values, val) + } + return values, nil + }, store) +} + +// Duration maps configuration directive to a time.Duration variable. +// +// Directive must be in form 'name duration' where duration is any string accepted by +// time.ParseDuration. As an additional requirement, result of time.ParseDuration must not +// be negative. +// +// Note that for convenience, if directive does have multiple arguments, they will be joined +// without separators. E.g. 'name 1h 2m' will become 'name 1h2m' and so '1h2m' will be passed +// to time.ParseDuration. +// +// See Map.Custom for description of arguments. +func (m *Map) Duration(name string, inheritGlobal, required bool, defaultVal time.Duration, store *time.Duration) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + if len(node.Args) == 0 { + return nil, NodeErr(node, "at least one argument is required") + } + + durationStr := strings.Join(node.Args, "") + dur, err := time.ParseDuration(durationStr) + if err != nil { + return nil, NodeErr(node, "%v", err) + } + + if dur < 0 { + return nil, NodeErr(node, "duration must not be negative") + } + + return dur, nil + }, store) +} + +func ParseDataSize(s string) (int, error) { + if len(s) == 0 { + return 0, errors.New("missing a number") + } + + // ' ' terminates the number+suffix pair. + s = s + " " + + var total int + currentDigit := "" + suffix := "" + for _, ch := range s { + if unicode.IsDigit(ch) { + if suffix != "" { + return 0, errors.New("unexpected digit after a suffix") + } + currentDigit += string(ch) + continue + } + if ch != ' ' { + suffix += string(ch) + continue + } + + num, err := strconv.Atoi(currentDigit) + if err != nil { + return 0, err + } + + if num < 0 { + return 0, errors.New("value must not be negative") + } + + switch suffix { + case "G": + total += num * 1024 * 1024 * 1024 + case "M": + total += num * 1024 * 1024 + case "K": + total += num * 1024 + case "B", "b": + total += num + default: + if num != 0 { + return 0, errors.New("unknown unit suffix: " + suffix) + } + } + + suffix = "" + currentDigit = "" + } + + return total, nil +} + +// DataSize maps configuration directive to a int variable, representing data size. +// +// Syntax requires unit suffix to be added to the end of string to specify +// data unit and allows multiple arguments (they will be added together). +// +// See Map.Custom for description of arguments. +func (m *Map) DataSize(name string, inheritGlobal, required bool, defaultVal int64, store *int64) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + if len(node.Args) == 0 { + return nil, NodeErr(node, "at least one argument is required") + } + + durationStr := strings.Join(node.Args, " ") + dur, err := ParseDataSize(durationStr) + if err != nil { + return nil, NodeErr(node, "%v", err) + } + + return int64(dur), nil + }, store) +} + +func ParseBool(s string) (bool, error) { + switch strings.ToLower(s) { + case "1", "true", "on", "yes": + return true, nil + case "0", "false", "off", "no": + return false, nil + } + return false, fmt.Errorf("bool argument should be 'yes' or 'no'") +} + +// Bool maps presence of some configuration directive to a boolean variable. +// Additionally, 'name yes' and 'name no' are mapped to true and false +// correspondingly. +// +// I.e. if directive 'io_debug' exists in processed configuration block or in +// the global configuration (if inheritGlobal is true) then Process will store +// true in target variable. +func (m *Map) Bool(name string, inheritGlobal, defaultVal bool, store *bool) { + m.Custom(name, inheritGlobal, false, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + if len(node.Args) == 0 { + return true, nil + } + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected exactly 1 argument") + } + + b, err := ParseBool(node.Args[0]) + if err != nil { + return nil, NodeErr(node, "bool argument should be 'yes' or 'no'") + } + return b, nil + }, store) +} + +// StringList maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name arbitrary_string arbitrary_string ...' +// Where at least one argument must be present. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) StringList(name string, inheritGlobal, required bool, defaultVal []string, store *[]string) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) == 0 { + return nil, NodeErr(node, "expected at least one argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + return node.Args, nil + }, store) +} + +// String maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name arbitrary_string'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) String(name string, inheritGlobal, required bool, defaultVal string, store *string) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + return node.Args[0], nil + }, store) +} + +// Int maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name 123'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) Int(name string, inheritGlobal, required bool, defaultVal int, store *int) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + i, err := strconv.Atoi(node.Args[0]) + if err != nil { + return nil, NodeErr(node, "invalid integer: %s", node.Args[0]) + } + return i, nil + }, store) +} + +// UInt maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name 123'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) UInt(name string, inheritGlobal, required bool, defaultVal uint, store *uint) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + i, err := strconv.ParseUint(node.Args[0], 10, 32) + if err != nil { + return nil, NodeErr(node, "invalid integer: %s", node.Args[0]) + } + return uint(i), nil + }, store) +} + +// Int32 maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name 123'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) Int32(name string, inheritGlobal, required bool, defaultVal int32, store *int32) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + i, err := strconv.ParseInt(node.Args[0], 10, 32) + if err != nil { + return nil, NodeErr(node, "invalid integer: %s", node.Args[0]) + } + return int32(i), nil + }, store) +} + +// UInt32 maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name 123'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) UInt32(name string, inheritGlobal, required bool, defaultVal uint32, store *uint32) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + i, err := strconv.ParseUint(node.Args[0], 10, 32) + if err != nil { + return nil, NodeErr(node, "invalid integer: %s", node.Args[0]) + } + return uint32(i), nil + }, store) +} + +// Int64 maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name 123'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) Int64(name string, inheritGlobal, required bool, defaultVal int64, store *int64) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + i, err := strconv.ParseInt(node.Args[0], 10, 64) + if err != nil { + return nil, NodeErr(node, "invalid integer: %s", node.Args[0]) + } + return i, nil + }, store) +} + +// UInt64 maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name 123'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) UInt64(name string, inheritGlobal, required bool, defaultVal uint64, store *uint64) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + i, err := strconv.ParseUint(node.Args[0], 10, 64) + if err != nil { + return nil, NodeErr(node, "invalid integer: %s", node.Args[0]) + } + return i, nil + }, store) +} + +// Float maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name 123.55'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) Float(name string, inheritGlobal, required bool, defaultVal float64, store *float64) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + + f, err := strconv.ParseFloat(node.Args[0], 64) + if err != nil { + return nil, NodeErr(node, "invalid float: %s", node.Args[0]) + } + return f, nil + }, store) +} + +// Custom maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// If inheritGlobal is true - Map will try to use a value from globalCfg if +// none is set in a processed configuration block. +// +// If required is true - Map will fail if no value is set in the configuration, +// both global (if inheritGlobal is true) and in the processed block. +// +// defaultVal is a factory function that should return the default value for +// the variable. It will be used if no value is set in the config. It can be +// nil if required is true. +// Note that if inheritGlobal is true, defaultVal of the global directive +// will be used instead. +// +// mapper is a function that should convert configuration directive arguments +// into variable value. Both functions may fail with errors, configuration +// processing will stop immediately then. +// Note: mapper function should not modify passed values. +// +// store is where the value returned by mapper should be stored. Can be nil +// (value will be saved only in Map.Values). +func (m *Map) Custom(name string, inheritGlobal, required bool, defaultVal func() (interface{}, error), mapper func(*Map, Node) (interface{}, error), store interface{}) { + if m.entries == nil { + m.entries = make(map[string]matcher) + } + if _, ok := m.entries[name]; ok { + panic("Map.Custom: duplicate matcher") + } + + var target *reflect.Value + ptr := reflect.ValueOf(store) + if ptr.IsValid() && !ptr.IsNil() { + val := ptr.Elem() + if !val.CanSet() { + panic("Map.Custom: store argument must be settable (a pointer)") + } + target = &val + } + + m.entries[name] = matcher{ + name: name, + inheritGlobal: inheritGlobal, + required: required, + defaultVal: defaultVal, + mapper: mapper, + store: target, + } +} + +// Callback creates mapping that will call mapper() function for each +// directive with the specified name. No further processing is done. +// +// Directives with the specified name will not be returned by Process if +// AllowUnknown is used. +// +// It is intended to permit multiple independent values of directive with +// implementation-defined handling. +func (m *Map) Callback(name string, mapper func(*Map, Node) error) { + if m.entries == nil { + m.entries = make(map[string]matcher) + } + if _, ok := m.entries[name]; ok { + panic("Map.Custom: duplicate matcher") + } + + m.entries[name] = matcher{ + name: name, + customCallback: mapper, + } +} + +// Process maps variables from global configuration and block passed in NewMap. +// +// If Map instance was not created using NewMap - Process panics. +func (m *Map) Process() (unknown []Node, err error) { + return m.ProcessWith(m.Globals, m.Block) +} + +// Process maps variables from global configuration and block passed in arguments. +func (m *Map) ProcessWith(globalCfg map[string]interface{}, block Node) (unknown []Node, err error) { + unknown = make([]Node, 0, len(block.Children)) + matched := make(map[string]bool) + m.Values = make(map[string]interface{}) + + for _, subnode := range block.Children { + matcher, ok := m.entries[subnode.Name] + if !ok { + if !m.allowUnknown { + return nil, NodeErr(subnode, "unexpected directive: %s", subnode.Name) + } + unknown = append(unknown, subnode) + continue + } + + if matcher.customCallback != nil { + if err := matcher.customCallback(m, subnode); err != nil { + return nil, err + } + matched[subnode.Name] = true + continue + } + + if matched[subnode.Name] { + return nil, NodeErr(subnode, "duplicate directive: %s", subnode.Name) + } + matched[subnode.Name] = true + + val, err := matcher.mapper(m, subnode) + if err != nil { + return nil, err + } + m.Values[matcher.name] = val + if matcher.store != nil { + matcher.assign(val) + } + } + + for _, matcher := range m.entries { + if matched[matcher.name] { + continue + } + if matcher.mapper == nil { + continue + } + + var val interface{} + globalVal, ok := globalCfg[matcher.name] + if matcher.inheritGlobal && ok { + val = globalVal + } else if !matcher.required { + if matcher.defaultVal == nil { + continue + } + + val, err = matcher.defaultVal() + if err != nil { + return nil, err + } + } else { + return nil, NodeErr(block, "missing required directive: %s", matcher.name) + } + + // If we put zero values into map then code that checks globalCfg + // above will inherit them for required fields instead of failing. + // + // This is important for fields that are required to be specified + // either globally or on per-block basis (e.g. tls, hostname). + // For these directives, global Map does have required = false + // so global values are default which is usually zero value. + // + // This is a temporary solutions, of course, in the long-term + // the way global values and "inheritance" is handled should be + // revised. + store := false + valT := reflect.TypeOf(val) + if valT != nil { + zero := reflect.Zero(valT) + store = !reflect.DeepEqual(val, zero.Interface()) + } + + if store { + m.Values[matcher.name] = val + } + if matcher.store != nil { + matcher.assign(val) + } + } + + return unknown, nil +} diff --git a/framework/config/map_test.go b/framework/config/map_test.go new file mode 100644 index 0000000..fd82630 --- /dev/null +++ b/framework/config/map_test.go @@ -0,0 +1,495 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package config + +import ( + "testing" +) + +func TestMapProcess(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"bar"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo := "" + m.Custom("foo", false, true, nil, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != "bar" { + t.Errorf("Incorrect value stored in variable, want 'bar', got '%s'", foo) + } +} + +func TestMapProcess_MissingRequired(t *testing.T) { + cfg := Node{ + Children: []Node{}, + } + + m := NewMap(nil, cfg) + + foo := "" + m.Custom("foo", false, true, nil, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err == nil { + t.Errorf("Expected failure") + } +} + +func TestMapProcess_InheritGlobal(t *testing.T) { + cfg := Node{ + Children: []Node{}, + } + + m := NewMap(map[string]interface{}{"foo": "bar"}, cfg) + + foo := "" + m.Custom("foo", true, true, nil, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != "bar" { + t.Errorf("Incorrect value stored in variable, want 'bar', got '%s'", foo) + } +} + +func TestMapProcess_InheritGlobal_MissingRequired(t *testing.T) { + cfg := Node{ + Children: []Node{}, + } + + m := NewMap(map[string]interface{}{}, cfg) + + foo := "" + m.Custom("foo", false, true, nil, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err == nil { + t.Errorf("Expected failure") + } +} + +func TestMapProcess_InheritGlobal_Override(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"bar"}, + }, + }, + } + + m := NewMap(map[string]interface{}{}, cfg) + + foo := "" + m.Custom("foo", false, true, nil, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != "bar" { + t.Errorf("Incorrect value stored in variable, want 'bar', got '%s'", foo) + } +} + +func TestMapProcess_DefaultValue(t *testing.T) { + cfg := Node{ + Children: []Node{}, + } + + m := NewMap(nil, cfg) + + foo := "" + m.Custom("foo", false, false, func() (interface{}, error) { + return "bar", nil + }, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != "bar" { + t.Errorf("Incorrect value stored in variable, want 'bar', got '%s'", foo) + } +} + +func TestMapProcess_InheritGlobal_DefaultValue(t *testing.T) { + cfg := Node{ + Children: []Node{}, + } + + m := NewMap(map[string]interface{}{"foo": "baz"}, cfg) + + foo := "" + m.Custom("foo", true, false, func() (interface{}, error) { + return "bar", nil + }, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != "baz" { + t.Errorf("Incorrect value stored in variable, want 'baz', got '%s'", foo) + } + + t.Run("no global", func(t *testing.T) { + _, err := m.ProcessWith(map[string]interface{}{}, cfg) + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != "bar" { + t.Errorf("Incorrect value stored in variable, want 'bar', got '%s'", foo) + } + }) +} + +func TestMapProcess_Duplicate(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"bar"}, + }, + { + Name: "foo", + Args: []string{"bar"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo := "" + m.Custom("foo", false, true, nil, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err == nil { + t.Errorf("Expected failure") + } +} + +func TestMapProcess_Unexpected(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"baz"}, + }, + { + Name: "bar", + Args: []string{"baz"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo := "" + m.Custom("bar", false, true, nil, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err == nil { + t.Errorf("Expected failure") + } + + m.AllowUnknown() + + unknown, err := m.Process() + if err != nil { + t.Errorf("Unexpected failure: %v", err) + } + + if len(unknown) != 1 { + t.Fatalf("Wrong amount of unknown nodes: %v", len(unknown)) + } + + if unknown[0].Name != "foo" { + t.Fatalf("Wrong node in unknown: %v", unknown[0].Name) + } +} + +func TestMapInt(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"1"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo := 0 + m.Int("foo", false, true, 0, &foo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != 1 { + t.Errorf("Incorrect value stored in variable, want 1, got %d", foo) + } +} + +func TestMapInt_Invalid(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"AAAA"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo := 0 + m.Int("foo", false, true, 0, &foo) + + _, err := m.Process() + if err == nil { + t.Errorf("Expected failure") + } +} + +func TestMapFloat(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"1"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo := 0.0 + m.Float("foo", false, true, 0, &foo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != 1.0 { + t.Errorf("Incorrect value stored in variable, want 1, got %v", foo) + } +} + +func TestMapFloat_Invalid(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"AAAA"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo := 0.0 + m.Float("foo", false, true, 0, &foo) + + _, err := m.Process() + if err == nil { + t.Errorf("Expected failure") + } +} + +func TestMapBool(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + }, + { + Name: "bar", + Args: []string{"yes"}, + }, + { + Name: "baz", + Args: []string{"no"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo, bar, baz, boo := false, false, false, false + m.Bool("foo", false, false, &foo) + m.Bool("bar", false, false, &bar) + m.Bool("baz", false, false, &baz) + m.Bool("boo", false, false, &boo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if !foo { + t.Errorf("Incorrect value stored in variable foo, want true, got false") + } + if !bar { + t.Errorf("Incorrect value stored in variable bar, want true, got false") + } + if baz { + t.Errorf("Incorrect value stored in variable baz, want false, got true") + } + if boo { + t.Errorf("Incorrect value stored in variable boo, want false, got true") + } +} + +func TestParseDataSize(t *testing.T) { + check := func(s string, ok bool, expected int) { + val, err := ParseDataSize(s) + if err != nil && ok { + t.Errorf("unexpected parseDataSize('%s') fail: %v", s, err) + return + } + if err == nil && !ok { + t.Errorf("unexpected parseDataSize('%s') success, got %d", s, val) + return + } + if val != expected { + t.Errorf("parseDataSize('%s') != %d", s, expected) + return + } + } + + check("1M", true, 1024*1024) + check("1K", true, 1024) + check("1b", true, 1) + check("1M 5b", true, 1024*1024+5) + check("1M 5K 5b", true, 1024*1024+5*1024+5) + check("0", true, 0) + check("1", false, 0) + check("1d", false, 0) + check("d", false, 0) + check("unrelated", false, 0) + check("1M5b", false, 0) + check("", false, 0) + check("-5M", false, 0) +} + +func TestMap_Callback(t *testing.T) { + called := map[string]int{} + + cfg := Node{ + Children: []Node{ + { + Name: "test2", + Args: []string{"a"}, + }, + { + Name: "test3", + Args: []string{"b"}, + }, + { + Name: "test3", + Args: []string{"b"}, + }, + { + Name: "unrelated", + Args: []string{"b"}, + }, + }, + } + m := NewMap(nil, cfg) + m.Callback("test1", func(*Map, Node) error { + called["test1"]++ + return nil + }) + m.Callback("test2", func(_ *Map, n Node) error { + called["test2"]++ + if n.Args[0] != "a" { + t.Fatal("Wrong n.Args[0] for test2:", n.Args[0]) + } + return nil + }) + m.Callback("test3", func(_ *Map, n Node) error { + called["test3"]++ + if n.Args[0] != "b" { + t.Fatal("Wrong n.Args[0] for test2:", n.Args[0]) + } + return nil + }) + m.AllowUnknown() + others, err := m.Process() + if err != nil { + t.Fatal("Unexpected error:", err) + } + if called["test1"] != 0 { + t.Error("test1 CB was called when it should not") + } + if called["test2"] != 1 { + t.Error("test2 CB was not called when it should") + } + if called["test3"] != 2 { + t.Error("test3 CB was not called when it should") + } + if len(others) != 1 { + t.Error("Wrong amount of unmatched directives") + } + if others[0].Name != "unrelated" { + t.Error("Wrong directive returned in unmatched slice:", others[0].Name) + } +} diff --git a/framework/config/module/check_action.go b/framework/config/module/check_action.go new file mode 100644 index 0000000..cac3278 --- /dev/null +++ b/framework/config/module/check_action.go @@ -0,0 +1,184 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package modconfig + +import ( + "errors" + "fmt" + "strconv" + "strings" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" +) + +// FailAction specifies actions that messages pipeline should take based on the +// result of the check. +// +// Its check module responsibility to apply FailAction on the CheckResult it +// returns. It is intended to be used as follows: +// +// Add the configuration directive to allow user to specify the action: +// +// cfg.Custom("SOME_action", false, false, +// func() (interface{}, error) { +// return modconfig.FailAction{Quarantine: true}, nil +// }, modconfig.FailActionDirective, &yourModule.SOMEAction) +// +// return in func literal is the default value, you might want to adjust it. +// +// Call yourModule.SOMEAction.Apply on CheckResult containing only the +// Reason field: +// +// func (yourModule YourModule) CheckConnection() module.CheckResult { +// return yourModule.SOMEAction.Apply(module.CheckResult{ +// Reason: ..., +// }) +// } +type FailAction struct { + Quarantine bool + Reject bool + + ReasonOverride *exterrors.SMTPError +} + +func FailActionDirective(_ *config.Map, node config.Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, config.NodeErr(node, "can't declare block here") + } + + val, err := ParseActionDirective(node.Args) + if err != nil { + return nil, config.NodeErr(node, "%v", err) + } + return val, nil +} + +func ParseActionDirective(args []string) (FailAction, error) { + if len(args) == 0 { + return FailAction{}, errors.New("expected at least 1 argument") + } + + res := FailAction{} + + switch args[0] { + case "reject", "quarantine": + if len(args) > 1 { + var err error + res.ReasonOverride, err = ParseRejectDirective(args[1:]) + if err != nil { + return FailAction{}, err + } + } + case "ignore": + default: + return FailAction{}, errors.New("invalid action") + } + + res.Reject = args[0] == "reject" + res.Quarantine = args[0] == "quarantine" + return res, nil +} + +// Apply merges the result of check execution with action configuration specified +// in the check configuration. +func (cfa FailAction) Apply(originalRes module.CheckResult) module.CheckResult { + if originalRes.Reason == nil { + return originalRes + } + + if cfa.ReasonOverride != nil { + // Wrap instead of replace to preserve other fields. + originalRes.Reason = &exterrors.SMTPError{ + Code: cfa.ReasonOverride.Code, + EnhancedCode: cfa.ReasonOverride.EnhancedCode, + Message: cfa.ReasonOverride.Message, + Err: originalRes.Reason, + } + } + + originalRes.Quarantine = cfa.Quarantine || originalRes.Quarantine + originalRes.Reject = cfa.Reject || originalRes.Reject + return originalRes +} + +func ParseRejectDirective(args []string) (*exterrors.SMTPError, error) { + code := 554 + enchCode := exterrors.EnhancedCode{0, 7, 0} + msg := "Message rejected due to a local policy" + var err error + switch len(args) { + case 3: + msg = args[2] + if msg == "" { + return nil, fmt.Errorf("message can't be empty") + } + fallthrough + case 2: + enchCode, err = parseEnhancedCode(args[1]) + if err != nil { + return nil, err + } + if enchCode[0] != 4 && enchCode[0] != 5 { + return nil, fmt.Errorf("enhanced code should use either 4 or 5 as a first number") + } + fallthrough + case 1: + code, err = strconv.Atoi(args[0]) + if err != nil { + return nil, fmt.Errorf("invalid error code integer: %v", err) + } + if (code/100) != 4 && (code/100) != 5 { + return nil, fmt.Errorf("error code should start with either 4 or 5") + } + // If enchanced code is not set - set first digit based on provided "basic" code. + if enchCode[0] == 0 { + enchCode[0] = code / 100 + } + case 0: + // If no codes provided at all - use 5.7.0 and 554. + enchCode[0] = 5 + default: + return nil, fmt.Errorf("invalid count of arguments") + } + return &exterrors.SMTPError{ + Code: code, + EnhancedCode: enchCode, + Message: msg, + Reason: "reject directive used", + }, nil +} + +func parseEnhancedCode(s string) (exterrors.EnhancedCode, error) { + parts := strings.Split(s, ".") + if len(parts) != 3 { + return exterrors.EnhancedCode{}, fmt.Errorf("wrong amount of enhanced code parts") + } + + code := exterrors.EnhancedCode{} + for i, part := range parts { + num, err := strconv.Atoi(part) + if err != nil { + return code, err + } + code[i] = num + } + return code, nil +} diff --git a/framework/config/module/interfaces.go b/framework/config/module/interfaces.go new file mode 100644 index 0000000..caf17c5 --- /dev/null +++ b/framework/config/module/interfaces.go @@ -0,0 +1,98 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package modconfig + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +func MessageCheck(globals map[string]interface{}, args []string, block config.Node) (module.Check, error) { + var check module.Check + if err := ModuleFromNode("check", args, block, globals, &check); err != nil { + return nil, err + } + return check, nil +} + +// DeliveryDirective is a callback for use in config.Map.Custom. +// +// It does all work necessary to create a module instance from the config +// directive with the following structure: +// +// directive_name mod_name [inst_name] [{ +// inline_mod_config +// }] +// +// Note that if used configuration structure lacks directive_name before mod_name - this function +// should not be used (call DeliveryTarget directly). +func DeliveryDirective(m *config.Map, node config.Node) (interface{}, error) { + return DeliveryTarget(m.Globals, node.Args, node) +} + +func DeliveryTarget(globals map[string]interface{}, args []string, block config.Node) (module.DeliveryTarget, error) { + var target module.DeliveryTarget + if err := ModuleFromNode("target", args, block, globals, &target); err != nil { + return nil, err + } + return target, nil +} + +func MsgModifier(globals map[string]interface{}, args []string, block config.Node) (module.Modifier, error) { + var check module.Modifier + if err := ModuleFromNode("modify", args, block, globals, &check); err != nil { + return nil, err + } + return check, nil +} + +func IMAPFilter(globals map[string]interface{}, args []string, block config.Node) (module.IMAPFilter, error) { + var filter module.IMAPFilter + if err := ModuleFromNode("imap.filter", args, block, globals, &filter); err != nil { + return nil, err + } + return filter, nil +} + +func StorageDirective(m *config.Map, node config.Node) (interface{}, error) { + var backend module.Storage + if err := ModuleFromNode("storage", node.Args, node, m.Globals, &backend); err != nil { + return nil, err + } + return backend, nil +} + +// Table is a convenience wrapper for TableDirective. +// +// cfg.Bool(...) +// modconfig.Table(cfg, "auth_map", false, false, nil, &mod.authMap) +// cfg.Process() +func Table(cfg *config.Map, name string, inheritGlobal, required bool, defaultVal module.Table, store *module.Table) { + cfg.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, TableDirective, store) +} + +func TableDirective(m *config.Map, node config.Node) (interface{}, error) { + var tbl module.Table + if err := ModuleFromNode("table", node.Args, node, m.Globals, &tbl); err != nil { + return nil, err + } + return tbl, nil +} diff --git a/framework/config/module/modconfig.go b/framework/config/module/modconfig.go new file mode 100644 index 0000000..3183bb9 --- /dev/null +++ b/framework/config/module/modconfig.go @@ -0,0 +1,163 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package modconfig provides matchers for config.Map that query +// modules registry and parse inline module definitions. +// +// They should be used instead of manual querying when there is need to +// reference a module instance in the configuration. +// +// See ModuleFromNode documentation for explanation of what is 'args' +// for some functions (DeliveryTarget). +package modconfig + +import ( + "fmt" + "io" + "reflect" + "strings" + + parser "github.com/foxcpp/maddy/framework/cfgparser" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +// createInlineModule is a helper function for config matchers that can create inline modules. +func createInlineModule(preferredNamespace, modName string, args []string) (module.Module, error) { + var newMod module.FuncNewModule + originalModName := modName + + // First try to extend the name with preferred namespace unless the name + // already contains it. + if !strings.Contains(modName, ".") && preferredNamespace != "" { + modName = preferredNamespace + "." + modName + newMod = module.Get(modName) + } + + // Then try global namespace for compatibility and complex modules. + if newMod == nil { + newMod = module.Get(originalModName) + } + + // Bail if both failed. + if newMod == nil { + return nil, fmt.Errorf("unknown module: %s (namespace: %s)", originalModName, preferredNamespace) + } + + return newMod(modName, "", nil, args) +} + +// initInlineModule constructs "faked" config tree and passes it to module +// Init function to make it look like it is defined at top-level. +// +// args must contain at least one argument, otherwise initInlineModule panics. +func initInlineModule(modObj module.Module, globals map[string]interface{}, block config.Node) error { + err := modObj.Init(config.NewMap(globals, block)) + if err != nil { + return err + } + + if closer, ok := modObj.(io.Closer); ok { + hooks.AddHook(hooks.EventShutdown, func() { + log.Debugf("close %s (%s)", modObj.Name(), modObj.InstanceName()) + if err := closer.Close(); err != nil { + log.Printf("module %s (%s) close failed: %v", modObj.Name(), modObj.InstanceName(), err) + } + }) + } + + return nil +} + +// ModuleFromNode does all work to create or get existing module object with a certain type. +// It is not used by top-level module definitions, only for references from other +// modules configuration blocks. +// +// inlineCfg should contain configuration directives for inline declarations. +// args should contain values that are used to create module. +// It should be either module name + instance name or just module name. Further extensions +// may add other string arguments (currently, they can be accessed by module instances +// as inlineArgs argument to constructor). +// +// It checks using reflection whether it is possible to store a module object into modObj +// pointer (e.g. it implements all necessary interfaces) and stores it if everything is fine. +// If module object doesn't implement necessary module interfaces - error is returned. +// If modObj is not a pointer, ModuleFromNode panics. +// +// preferredNamespace is used as an implicit prefix for module name lookups. +// Module with name preferredNamespace + "." + args[0] will be preferred over just args[0]. +// It can be omitted. +func ModuleFromNode(preferredNamespace string, args []string, inlineCfg config.Node, globals map[string]interface{}, moduleIface interface{}) error { + if len(args) == 0 { + return parser.NodeErr(inlineCfg, "at least one argument is required") + } + + referenceExisting := strings.HasPrefix(args[0], "&") + + var modObj module.Module + var err error + if referenceExisting { + if len(args) != 1 || inlineCfg.Children != nil { + return parser.NodeErr(inlineCfg, "exactly one argument is required to use existing config block") + } + modObj, err = module.GetInstance(args[0][1:]) + log.Debugf("%s:%d: reference %s", inlineCfg.File, inlineCfg.Line, args[0]) + } else { + log.Debugf("%s:%d: new module %s %v", inlineCfg.File, inlineCfg.Line, args[0], args[1:]) + modObj, err = createInlineModule(preferredNamespace, args[0], args[1:]) + } + if err != nil { + return err + } + + // NOTE: This will panic if moduleIface is not a pointer. + modIfaceType := reflect.TypeOf(moduleIface).Elem() + modObjType := reflect.TypeOf(modObj) + + if modIfaceType.Kind() == reflect.Interface { + // Case for assignment to module interface type. + if !modObjType.Implements(modIfaceType) && !modObjType.AssignableTo(modIfaceType) { + return parser.NodeErr(inlineCfg, "module %s (%s) doesn't implement %v interface", modObj.Name(), modObj.InstanceName(), modIfaceType) + } + } else if !modObjType.AssignableTo(modIfaceType) { + // Case for assignment to concrete module type. Used in "module groups". + return parser.NodeErr(inlineCfg, "module %s (%s) is not %v", modObj.Name(), modObj.InstanceName(), modIfaceType) + } + + reflect.ValueOf(moduleIface).Elem().Set(reflect.ValueOf(modObj)) + + if !referenceExisting { + if err := initInlineModule(modObj, globals, inlineCfg); err != nil { + return err + } + } + + return nil +} + +// GroupFromNode provides a special kind of ModuleFromNode syntax that allows +// to omit the module name when defining inine configuration. If it is not +// present, name in defaultModule is used. +func GroupFromNode(defaultModule string, args []string, inlineCfg config.Node, globals map[string]interface{}, moduleIface interface{}) error { + if len(args) == 0 { + args = append(args, defaultModule) + } + return ModuleFromNode("", args, inlineCfg, globals, moduleIface) +} diff --git a/framework/config/tls/client.go b/framework/config/tls/client.go new file mode 100644 index 0000000..cf21b3c --- /dev/null +++ b/framework/config/tls/client.go @@ -0,0 +1,88 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tls + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" +) + +func TLSClientBlock(_ *config.Map, node config.Node) (interface{}, error) { + cfg := tls.Config{} + + childM := config.NewMap(nil, node) + var ( + tlsVersions [2]uint16 + rootCAPaths []string + certPath, keyPath string + ) + + childM.StringList("root_ca", false, false, nil, &rootCAPaths) + childM.String("cert", false, false, "", &certPath) + childM.String("key", false, false, "", &keyPath) + childM.Custom("protocols", false, false, func() (interface{}, error) { + return [2]uint16{0, 0}, nil + }, TLSVersionsDirective, &tlsVersions) + childM.Custom("ciphers", false, false, func() (interface{}, error) { + return nil, nil + }, TLSCiphersDirective, &cfg.CipherSuites) + childM.Custom("curves", false, false, func() (interface{}, error) { + return nil, nil + }, TLSCurvesDirective, &cfg.CurvePreferences) + + if _, err := childM.Process(); err != nil { + return nil, err + } + + if len(rootCAPaths) != 0 { + pool := x509.NewCertPool() + for _, path := range rootCAPaths { + blob, err := os.ReadFile(path) + if err != nil { + return nil, err + } + if !pool.AppendCertsFromPEM(blob) { + return nil, fmt.Errorf("no certificates was loaded from %s", path) + } + } + cfg.RootCAs = pool + } + + if certPath != "" && keyPath != "" { + keypair, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return nil, err + } + log.Debugf("using client keypair %s/%s", certPath, keyPath) + cfg.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return &keypair, nil + } + } + + cfg.MinVersion = tlsVersions[0] + cfg.MaxVersion = tlsVersions[1] + log.Debugf("tls: min version: %x, max version: %x", tlsVersions[0], tlsVersions[1]) + + return &cfg, nil +} diff --git a/framework/config/tls/general.go b/framework/config/tls/general.go new file mode 100644 index 0000000..a3fd953 --- /dev/null +++ b/framework/config/tls/general.go @@ -0,0 +1,136 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tls + +import ( + "crypto/tls" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" +) + +var strVersionsMap = map[string]uint16{ + "tls1.0": tls.VersionTLS10, + "tls1.1": tls.VersionTLS11, + "tls1.2": tls.VersionTLS12, + "tls1.3": tls.VersionTLS13, + "": 0, // use crypto/tls defaults if value is not specified +} + +var strCiphersMap = map[string]uint16{ + // TLS 1.0 - 1.2 cipher suites. + "RSA-WITH-RC4128-SHA": tls.TLS_RSA_WITH_RC4_128_SHA, + "RSA-WITH-3DES-EDE-CBC-SHA": tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, + "RSA-WITH-AES128-CBC-SHA": tls.TLS_RSA_WITH_AES_128_CBC_SHA, + "RSA-WITH-AES256-CBC-SHA": tls.TLS_RSA_WITH_AES_256_CBC_SHA, + "RSA-WITH-AES128-CBC-SHA256": tls.TLS_RSA_WITH_AES_128_CBC_SHA256, + "RSA-WITH-AES128-GCM-SHA256": tls.TLS_RSA_WITH_AES_128_GCM_SHA256, + "RSA-WITH-AES256-GCM-SHA384": tls.TLS_RSA_WITH_AES_256_GCM_SHA384, + "ECDHE-ECDSA-WITH-RC4128-SHA": tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, + "ECDHE-ECDSA-WITH-AES128-CBC-SHA": tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + "ECDHE-ECDSA-WITH-AES256-CBC-SHA": tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + "ECDHE-RSA-WITH-RC4128-SHA": tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, + "ECDHE-RSA-WITH-3DES-EDE-CBC-SHA": tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, + "ECDHE-RSA-WITH-AES128-CBC-SHA": tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + "ECDHE-RSA-WITH-AES256-CBC-SHA": tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + "ECDHE-ECDSA-WITH-AES128-CBC-SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, + "ECDHE-RSA-WITH-AES128-CBC-SHA256": tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, + "ECDHE-RSA-WITH-AES128-GCM-SHA256": tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + "ECDHE-ECDSA-WITH-AES128-GCM-SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + "ECDHE-RSA-WITH-AES256-GCM-SHA384": tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + "ECDHE-ECDSA-WITH-AES256-GCM-SHA384": tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + "ECDHE-RSA-WITH-CHACHA20-POLY1305": tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + "ECDHE-ECDSA-WITH-CHACHA20-POLY1305": tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, +} + +var strCurvesMap = map[string]tls.CurveID{ + "p256": tls.CurveP256, + "p384": tls.CurveP384, + "p521": tls.CurveP521, + "X25519": tls.X25519, +} + +// TLSversionsDirective parses directive with arguments that specify +// minimum and maximum supported TLS versions. +// +// It returns [2]uint16 value for use in corresponding fields from tls.Config. +func TLSVersionsDirective(_ *config.Map, node config.Node) (interface{}, error) { + switch len(node.Args) { + case 1: + value, ok := strVersionsMap[node.Args[0]] + if !ok { + return nil, config.NodeErr(node, "invalid TLS version value: %s", node.Args[0]) + } + return [2]uint16{value, value}, nil + case 2: + minValue, ok := strVersionsMap[node.Args[0]] + if !ok { + return nil, config.NodeErr(node, "invalid TLS version value: %s", node.Args[0]) + } + maxValue, ok := strVersionsMap[node.Args[1]] + if !ok { + return nil, config.NodeErr(node, "invalid TLS version value: %s", node.Args[1]) + } + return [2]uint16{minValue, maxValue}, nil + default: + return nil, config.NodeErr(node, "expected 1 or 2 arguments") + } +} + +// TLSCiphersDirective parses directive with arguments that specify +// list of ciphers to offer to clients (or to use for outgoing connections). +// +// It returns list of []uint16 with corresponding cipher IDs. +func TLSCiphersDirective(_ *config.Map, node config.Node) (interface{}, error) { + if len(node.Args) == 0 { + return nil, config.NodeErr(node, "expected at least 1 argument, got 0") + } + + res := make([]uint16, 0, len(node.Args)) + for _, arg := range node.Args { + cipherId, ok := strCiphersMap[arg] + if !ok { + return nil, config.NodeErr(node, "unknown cipher: %s", arg) + } + res = append(res, cipherId) + } + log.Debugln("tls: using non-default cipherset:", node.Args) + return res, nil +} + +// TLSCurvesDirective parses directive with arguments that specify +// elliptic curves to use during TLS key exchange. +// +// It returns []tls.CurveID. +func TLSCurvesDirective(_ *config.Map, node config.Node) (interface{}, error) { + if len(node.Args) == 0 { + return nil, config.NodeErr(node, "expected at least 1 argument, got 0") + } + + res := make([]tls.CurveID, 0, len(node.Args)) + for _, arg := range node.Args { + curveId, ok := strCurvesMap[arg] + if !ok { + return nil, config.NodeErr(node, "unknown curve: %s", arg) + } + res = append(res, curveId) + } + log.Debugln("tls: using non-default curve preferences:", node.Args) + return res, nil +} diff --git a/framework/config/tls/server.go b/framework/config/tls/server.go new file mode 100644 index 0000000..4fe8e8d --- /dev/null +++ b/framework/config/tls/server.go @@ -0,0 +1,124 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tls + +import ( + "crypto/tls" + + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +type TLSConfig struct { + loader module.TLSLoader + baseCfg *tls.Config +} + +func (cfg *TLSConfig) Get() (*tls.Config, error) { + if cfg.loader == nil { + return nil, nil + } + tlsCfg := cfg.baseCfg.Clone() + + err := cfg.loader.ConfigureTLS(tlsCfg) + if err != nil { + return nil, err + } + + return tlsCfg, nil +} + +// TLSDirective reads the TLS configuration and adds the reload handler to +// reread certificates on SIGUSR2. +// +// The returned value is *tls.Config with GetConfigForClient set. +// If the 'tls off' is used, returned value is nil. +func TLSDirective(m *config.Map, node config.Node) (interface{}, error) { + cfg, err := readTLSBlock(m.Globals, node) + if err != nil { + return nil, err + } + + if cfg == nil { + return nil, nil + } + + return &tls.Config{ + GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + return cfg.Get() + }, + }, nil +} + +func readTLSBlock(globals map[string]interface{}, blockNode config.Node) (*TLSConfig, error) { + baseCfg := tls.Config{ + // Workaround for issue https://github.com/foxcpp/maddy/issues/730 + SessionTicketsDisabled: true, + } + + var loader module.TLSLoader + if len(blockNode.Args) > 0 { + if blockNode.Args[0] == "off" { + return nil, nil + } + + err := modconfig.ModuleFromNode("tls.loader", blockNode.Args, config.Node{}, globals, &loader) + if err != nil { + return nil, err + } + } + + childM := config.NewMap(globals, blockNode) + var tlsVersions [2]uint16 + + childM.Custom("loader", false, false, func() (interface{}, error) { + return loader, nil + }, func(_ *config.Map, node config.Node) (interface{}, error) { + var l module.TLSLoader + err := modconfig.ModuleFromNode("tls.loader", node.Args, node, globals, &l) + return l, err + }, &loader) + + childM.Custom("protocols", false, false, func() (interface{}, error) { + return [2]uint16{tls.VersionTLS10, 0}, nil + }, TLSVersionsDirective, &tlsVersions) + + childM.Custom("ciphers", false, false, func() (interface{}, error) { + return nil, nil + }, TLSCiphersDirective, &baseCfg.CipherSuites) + + childM.Custom("curves", false, false, func() (interface{}, error) { + return nil, nil + }, TLSCurvesDirective, &baseCfg.CurvePreferences) + + if _, err := childM.Process(); err != nil { + return nil, err + } + + baseCfg.MinVersion = tlsVersions[0] + baseCfg.MaxVersion = tlsVersions[1] + log.Debugf("tls: min version: %x, max version: %x", tlsVersions[0], tlsVersions[1]) + + return &TLSConfig{ + loader: loader, + baseCfg: &baseCfg, + }, nil +} diff --git a/framework/dns/debugflags.go b/framework/dns/debugflags.go new file mode 100644 index 0000000..fde218b --- /dev/null +++ b/framework/dns/debugflags.go @@ -0,0 +1,36 @@ +//go:build debugflags +// +build debugflags + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dns + +import ( + maddycli "github.com/foxcpp/maddy/internal/cli" + "github.com/urfave/cli/v2" +) + +func init() { + maddycli.AddGlobalFlag(&cli.StringFlag{ + Name: "debug.dnsoverride", + Usage: "replace the DNS resolver address", + Value: "system-default", + Destination: &overrideServ, + }) +} diff --git a/framework/dns/dnssec.go b/framework/dns/dnssec.go new file mode 100644 index 0000000..b8e9c19 --- /dev/null +++ b/framework/dns/dnssec.go @@ -0,0 +1,403 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dns + +import ( + "context" + "net" + "strconv" + "strings" + "time" + + "github.com/foxcpp/maddy/framework/log" + "github.com/miekg/dns" +) + +type TLSA = dns.TLSA + +// ExtResolver is a convenience wrapper for miekg/dns library that provides +// access to certain low-level functionality (notably, AD flag in responses, +// indicating whether DNSSEC verification was performed by the server). +type ExtResolver struct { + cl *dns.Client + Cfg *dns.ClientConfig +} + +// RCodeError is returned by ExtResolver when the RCODE in response is not +// NOERROR. +type RCodeError struct { + Name string + Code int +} + +func (err RCodeError) Temporary() bool { + return err.Code == dns.RcodeServerFailure +} + +func (err RCodeError) Error() string { + switch err.Code { + case dns.RcodeFormatError: + return "dns: rcode FORMERR when looking up " + err.Name + case dns.RcodeServerFailure: + return "dns: rcode SERVFAIL when looking up " + err.Name + case dns.RcodeNameError: + return "dns: rcode NXDOMAIN when looking up " + err.Name + case dns.RcodeNotImplemented: + return "dns: rcode NOTIMP when looking up " + err.Name + case dns.RcodeRefused: + return "dns: rcode REFUSED when looking up " + err.Name + } + return "dns: non-success rcode: " + strconv.Itoa(err.Code) + " when looking up " + err.Name +} + +func IsNotFound(err error) bool { + if dnsErr, ok := err.(*net.DNSError); ok { + return dnsErr.IsNotFound + } + if rcodeErr, ok := err.(RCodeError); ok { + return rcodeErr.Code == dns.RcodeNameError + } + return false +} + +func isLoopback(addr string) bool { + ip := net.ParseIP(addr) + if ip == nil { + return false + } + return ip.IsLoopback() +} + +func (e ExtResolver) exchange(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + var resp *dns.Msg + var lastErr error + for _, srv := range e.Cfg.Servers { + resp, _, lastErr = e.cl.ExchangeContext(ctx, msg, net.JoinHostPort(srv, e.Cfg.Port)) + if lastErr != nil { + continue + } + + if resp.Rcode != dns.RcodeSuccess { + lastErr = RCodeError{msg.Question[0].Name, resp.Rcode} + continue + } + + // Diregard AD flags from non-local resolvers, likely they are + // communicated with using an insecure channel and so flags can be + // tampered with. + if !isLoopback(srv) { + resp.AuthenticatedData = false + } + + break + } + return resp, lastErr +} + +func (e ExtResolver) AuthLookupAddr(ctx context.Context, addr string) (ad bool, names []string, err error) { + revAddr, err := dns.ReverseAddr(addr) + if err != nil { + return false, nil, err + } + + msg := new(dns.Msg) + msg.SetQuestion(revAddr, dns.TypePTR) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + + resp, err := e.exchange(ctx, msg) + if err != nil { + return false, nil, err + } + + ad = resp.AuthenticatedData + names = make([]string, 0, len(resp.Answer)) + for _, rr := range resp.Answer { + ptrRR, ok := rr.(*dns.PTR) + if !ok { + continue + } + + names = append(names, ptrRR.Ptr) + } + return +} + +func (e ExtResolver) AuthLookupHost(ctx context.Context, host string) (ad bool, addrs []string, err error) { + ad, addrParsed, err := e.AuthLookupIPAddr(ctx, host) + if err != nil { + return false, nil, err + } + + addrs = make([]string, 0, len(addrParsed)) + for _, addr := range addrParsed { + addrs = append(addrs, addr.String()) + } + return ad, addrs, nil +} + +func (e ExtResolver) AuthLookupMX(ctx context.Context, name string) (ad bool, mxs []*net.MX, err error) { + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(name), dns.TypeMX) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + + resp, err := e.exchange(ctx, msg) + if err != nil { + return false, nil, err + } + + ad = resp.AuthenticatedData + mxs = make([]*net.MX, 0, len(resp.Answer)) + for _, rr := range resp.Answer { + mxRR, ok := rr.(*dns.MX) + if !ok { + continue + } + + mxs = append(mxs, &net.MX{ + Host: mxRR.Mx, + Pref: mxRR.Preference, + }) + } + return +} + +func (e ExtResolver) AuthLookupTXT(ctx context.Context, name string) (ad bool, recs []string, err error) { + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(name), dns.TypeTXT) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + + resp, err := e.exchange(ctx, msg) + if err != nil { + return false, nil, err + } + + ad = resp.AuthenticatedData + recs = make([]string, 0, len(resp.Answer)) + for _, rr := range resp.Answer { + txtRR, ok := rr.(*dns.TXT) + if !ok { + continue + } + + recs = append(recs, strings.Join(txtRR.Txt, "")) + } + return +} + +// CheckCNAMEAD is a special function for use in DANE lookups. It attempts to determine final +// (canonical) name of the host and also reports whether the whole chain of CNAME's and final zone +// are "secure". +// +// If there are no A or AAAA records for host, rname = "" is returned. +func (e ExtResolver) CheckCNAMEAD(ctx context.Context, host string) (ad bool, rname string, err error) { + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(host), dns.TypeA) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + resp, err := e.exchange(ctx, msg) + if err != nil { + return false, "", err + } + + for _, r := range resp.Answer { + switch r := r.(type) { + case *dns.A: + rname = r.Hdr.Name + ad = resp.AuthenticatedData // Use AD flag from response we used to determine rname + } + } + + if rname == "" { + // IPv6-only host? Try to find out rname using AAAA lookup. + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(host), dns.TypeAAAA) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + resp, err := e.exchange(ctx, msg) + if err == nil { + for _, r := range resp.Answer { + switch r := r.(type) { + case *dns.AAAA: + rname = r.Hdr.Name + ad = resp.AuthenticatedData + } + } + } + } + + return ad, rname, nil +} + +func (e ExtResolver) AuthLookupCNAME(ctx context.Context, host string) (ad bool, cname string, err error) { + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(host), dns.TypeCNAME) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + resp, err := e.exchange(ctx, msg) + if err != nil { + return false, "", err + } + + for _, r := range resp.Answer { + cnameR, ok := r.(*dns.CNAME) + if !ok { + continue + } + return resp.AuthenticatedData, cnameR.Target, nil + } + + return resp.AuthenticatedData, "", nil +} + +func (e ExtResolver) AuthLookupIPAddr(ctx context.Context, host string) (ad bool, addrs []net.IPAddr, err error) { + // First, query IPv6. + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(host), dns.TypeAAAA) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + + resp, err := e.exchange(ctx, msg) + aaaaFailed := false + var ( + v6ad bool + v6addrs []net.IPAddr + ) + if err != nil { + // Disregard the error for AAAA lookups. + aaaaFailed = true + log.DefaultLogger.Error("Network I/O error during AAAA lookup", err, "host", host) + } else { + v6addrs = make([]net.IPAddr, 0, len(resp.Answer)) + v6ad = resp.AuthenticatedData + for _, rr := range resp.Answer { + aaaaRR, ok := rr.(*dns.AAAA) + if !ok { + continue + } + v6addrs = append(v6addrs, net.IPAddr{IP: aaaaRR.AAAA}) + } + } + + // Then repeat query with IPv4. + msg = new(dns.Msg) + msg.SetQuestion(dns.Fqdn(host), dns.TypeA) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + + resp, err = e.exchange(ctx, msg) + var ( + v4ad bool + v4addrs []net.IPAddr + ) + if err != nil { + if aaaaFailed { + return false, nil, err + } + // Disregard A lookup error if AAAA succeeded. + log.DefaultLogger.Error("Network I/O error during A lookup, using AAAA records", err, "host", host) + } else { + v4ad = resp.AuthenticatedData + v4addrs = make([]net.IPAddr, 0, len(resp.Answer)) + for _, rr := range resp.Answer { + aRR, ok := rr.(*dns.A) + if !ok { + continue + } + v4addrs = append(v4addrs, net.IPAddr{IP: aRR.A}) + } + } + + // A little bit of careful handling is required if AD is inconsistent + // for A and AAAA queries. This unfortunatenly happens in practice. For + // purposes of DANE handling (A/AAAA check) we disregard AAAA records + // if they are not authenctiated and return only A records with AD=true. + + addrs = make([]net.IPAddr, 0, len(v4addrs)+len(v6addrs)) + if !v6ad && !v4ad { + addrs = append(addrs, v6addrs...) + addrs = append(addrs, v4addrs...) + } else { + if v6ad { + addrs = append(addrs, v6addrs...) + } + addrs = append(addrs, v4addrs...) + } + return v4ad, addrs, nil +} + +func (e ExtResolver) AuthLookupTLSA(ctx context.Context, service, network, domain string) (ad bool, recs []TLSA, err error) { + name, err := dns.TLSAName(domain, service, network) + if err != nil { + return false, nil, err + } + + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(name), dns.TypeTLSA) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + + resp, err := e.exchange(ctx, msg) + if err != nil { + return false, nil, err + } + + ad = resp.AuthenticatedData + recs = make([]dns.TLSA, 0, len(resp.Answer)) + for _, rr := range resp.Answer { + rr, ok := rr.(*dns.TLSA) + if !ok { + continue + } + + recs = append(recs, *rr) + } + return +} + +func NewExtResolver() (*ExtResolver, error) { + cfg, err := dns.ClientConfigFromFile("/etc/resolv.conf") + if err != nil { + return nil, err + } + + if overrideServ != "" && overrideServ != "system-default" { + host, port, err := net.SplitHostPort(overrideServ) + if err != nil { + panic(err) + } + cfg.Servers = []string{host} + cfg.Port = port + } + + if len(cfg.Servers) == 0 { + cfg.Servers = []string{"127.0.0.1"} + } + + cl := new(dns.Client) + cl.Dialer = &net.Dialer{ + Timeout: time.Duration(cfg.Timeout) * time.Second, + } + return &ExtResolver{ + cl: cl, + Cfg: cfg, + }, nil +} diff --git a/framework/dns/dnssec_test.go b/framework/dns/dnssec_test.go new file mode 100644 index 0000000..774897b --- /dev/null +++ b/framework/dns/dnssec_test.go @@ -0,0 +1,196 @@ +package dns + +import ( + "context" + "fmt" + "net" + "reflect" + "strconv" + "testing" + "time" + + "github.com/foxcpp/maddy/framework/log" + "github.com/miekg/dns" +) + +type TestSrvAction int + +const ( + TestSrvTimeout TestSrvAction = iota + TestSrvServfail + TestSrvNoAddr + TestSrvOk +) + +func (a TestSrvAction) String() string { + switch a { + case TestSrvTimeout: + return "SrvTimeout" + case TestSrvServfail: + return "SrvServfail" + case TestSrvNoAddr: + return "SrvNoAddr" + case TestSrvOk: + return "SrvOk" + default: + panic("wtf action") + } +} + +type IPAddrTestServer struct { + udpServ dns.Server + aAction TestSrvAction + aAD bool + aaaaAction TestSrvAction + aaaaAD bool +} + +func (s *IPAddrTestServer) Run() { + pconn, err := net.ListenPacket("udp4", "127.0.0.1:0") + if err != nil { + panic(err) + } + s.udpServ.PacketConn = pconn + s.udpServ.Handler = s + go s.udpServ.ActivateAndServe() //nolint:errcheck +} + +func (s *IPAddrTestServer) Close() { + s.udpServ.PacketConn.Close() +} + +func (s *IPAddrTestServer) Addr() *net.UDPAddr { + return s.udpServ.PacketConn.LocalAddr().(*net.UDPAddr) +} + +func (s *IPAddrTestServer) ServeDNS(w dns.ResponseWriter, m *dns.Msg) { + q := m.Question[0] + + var ( + act TestSrvAction + ad bool + ) + switch q.Qtype { + case dns.TypeA: + act = s.aAction + ad = s.aAD + case dns.TypeAAAA: + act = s.aaaaAction + ad = s.aaaaAD + default: + panic("wtf qtype") + } + + reply := new(dns.Msg) + reply.SetReply(m) + reply.RecursionAvailable = true + reply.AuthenticatedData = ad + + switch act { + case TestSrvTimeout: + return // no nobody heard from him since... + case TestSrvServfail: + reply.Rcode = dns.RcodeServerFailure + case TestSrvNoAddr: + case TestSrvOk: + switch q.Qtype { + case dns.TypeA: + reply.Answer = append(reply.Answer, &dns.A{ + Hdr: dns.RR_Header{ + Name: q.Name, + Rrtype: dns.TypeA, + Class: dns.ClassINET, + Ttl: 9999, + }, + A: net.ParseIP("127.0.0.1"), + }) + case dns.TypeAAAA: + reply.Answer = append(reply.Answer, &dns.AAAA{ + Hdr: dns.RR_Header{ + Name: q.Name, + Rrtype: dns.TypeAAAA, + Class: dns.ClassINET, + Ttl: 9999, + }, + AAAA: net.ParseIP("::1"), + }) + } + } + + if err := w.WriteMsg(reply); err != nil { + panic(err) + } +} + +func TestExtResolver_AuthLookupIPAddr(t *testing.T) { + // AuthLookupIPAddr has a rather convoluted logic for combined A/AAAA + // lookups that return the best-effort result and also has some nuanced in + // AD flag handling for use in DANE algorithms. + + // Silence log messages about disregarded I/O errors. + log.DefaultLogger.Out = nil + + test := func(aAct, aaaaAct TestSrvAction, aAD, aaaaAD, ad bool, addrs []net.IP, err bool) { + t.Helper() + t.Run(fmt.Sprintln(aAct, aaaaAct, aAD, aaaaAD), func(t *testing.T) { + t.Helper() + + s := IPAddrTestServer{} + s.aAction = aAct + s.aaaaAction = aaaaAct + s.aAD = aAD + s.aaaaAD = aaaaAD + s.Run() + defer s.Close() + res := ExtResolver{ + cl: new(dns.Client), + Cfg: &dns.ClientConfig{ + Servers: []string{"127.0.0.1"}, + Port: strconv.Itoa(s.Addr().Port), + Timeout: 1, + }, + } + res.cl.Dialer = &net.Dialer{ + Timeout: 500 * time.Millisecond, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + actualAd, actualAddrs, actualErr := res.AuthLookupIPAddr(ctx, "maddy.test") + if (actualErr != nil) != err { + t.Fatal("actualErr:", actualErr, "expectedErr:", err) + } + if actualAd != ad { + t.Error("actualAd:", actualAd, "expectedAd:", ad) + } + ipAddrs := make([]net.IPAddr, 0, len(addrs)) + if len(addrs) == 0 { + ipAddrs = nil // lookup returns nil addrs for error cases + } + for _, a := range addrs { + ipAddrs = append(ipAddrs, net.IPAddr{IP: a, Zone: ""}) + } + if !reflect.DeepEqual(actualAddrs, ipAddrs) { + t.Logf("actualAddrs: %#+v", actualAddrs) + t.Logf("addrs: %#+v", ipAddrs) + t.Fail() + } + }) + } + + test(TestSrvOk, TestSrvOk, true, true, true, []net.IP{net.ParseIP("::1"), net.ParseIP("127.0.0.1").To4()}, false) + test(TestSrvOk, TestSrvOk, true, false, true, []net.IP{net.ParseIP("127.0.0.1").To4()}, false) + test(TestSrvOk, TestSrvOk, false, true, false, []net.IP{net.ParseIP("::1"), net.ParseIP("127.0.0.1").To4()}, false) + test(TestSrvOk, TestSrvOk, false, false, false, []net.IP{net.ParseIP("::1"), net.ParseIP("127.0.0.1").To4()}, false) + test(TestSrvOk, TestSrvTimeout, true, true, true, []net.IP{net.ParseIP("127.0.0.1").To4()}, false) + test(TestSrvOk, TestSrvServfail, true, true, true, []net.IP{net.ParseIP("127.0.0.1").To4()}, false) + test(TestSrvOk, TestSrvNoAddr, true, true, true, []net.IP{net.ParseIP("127.0.0.1").To4()}, false) + test(TestSrvNoAddr, TestSrvOk, true, true, true, []net.IP{net.ParseIP("::1")}, false) + test(TestSrvServfail, TestSrvServfail, true, true, false, nil, true) + + // actualAd is false, we don't want to risk reporting positive AD result if + // something is wrong with IPv4 lookup. + test(TestSrvTimeout, TestSrvOk, true, true, false, []net.IP{net.ParseIP("::1")}, false) + test(TestSrvServfail, TestSrvOk, true, true, false, []net.IP{net.ParseIP("::1")}, false) +} diff --git a/framework/dns/idna.go b/framework/dns/idna.go new file mode 100644 index 0000000..3592c42 --- /dev/null +++ b/framework/dns/idna.go @@ -0,0 +1,37 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dns + +import ( + "golang.org/x/net/idna" + "golang.org/x/text/unicode/norm" +) + +// SelectIDNA is a convenience function for encoding to/from Punycode. +// +// If ulabel is true, it returns U-label encoded domain in the Unicode NFC +// form. +// If ulabel is false, it returns A-label encoded domain. +func SelectIDNA(ulabel bool, domain string) (string, error) { + if ulabel { + uDomain, err := idna.ToUnicode(domain) + return norm.NFC.String(uDomain), err + } + return idna.ToASCII(domain) +} diff --git a/framework/dns/norm.go b/framework/dns/norm.go new file mode 100644 index 0000000..6d236e9 --- /dev/null +++ b/framework/dns/norm.go @@ -0,0 +1,72 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dns + +import ( + "strings" + + "github.com/miekg/dns" + "golang.org/x/net/idna" + "golang.org/x/text/unicode/norm" +) + +func FQDN(domain string) string { + return dns.Fqdn(domain) +} + +// ForLookup converts the domain into a canonical form suitable for table +// lookups and other comparisons. +// +// TL;DR Use this instead of strings.ToLower to prepare domain for lookups. +// +// Domains that contain invalid UTF-8 or invalid A-label +// domains are simply converted to local-case using strings.ToLower, but the +// error is also returned. +func ForLookup(domain string) (string, error) { + uDomain, err := idna.ToUnicode(domain) + if err != nil { + return strings.ToLower(domain), err + } + + // Side note: strings.ToLower does not support full case-folding, so it is + // important to apply NFC normalization first. + uDomain = norm.NFC.String(uDomain) + uDomain = strings.ToLower(uDomain) + uDomain = strings.TrimSuffix(uDomain, ".") + return uDomain, nil +} + +// Equal reports whether domain1 and domain2 are equivalent as defined by +// IDNA2008 (RFC 5890). +// +// TL;DR Use this instead of strings.EqualFold to compare domains. +// +// Equivalence for malformed A-label domains is defined using regular +// byte-string comparison with case-folding applied. +func Equal(domain1, domain2 string) bool { + // Short circult. If they are bit-equivalent, then they are also semantically + // equivalent. + if domain1 == domain2 { + return true + } + + uDomain1, _ := ForLookup(domain1) + uDomain2, _ := ForLookup(domain2) + return uDomain1 == uDomain2 +} diff --git a/framework/dns/override.go b/framework/dns/override.go new file mode 100644 index 0000000..25f0da0 --- /dev/null +++ b/framework/dns/override.go @@ -0,0 +1,53 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dns + +import ( + "context" + "net" + "time" +) + +var overrideServ string + +// override globally overrides the used DNS server address with one provided. +// This function is meant only for testing. It should be called before any modules are +// initialized to have full effect. +// +// The server argument is in form of "IP:PORT". It is expected that the server +// will be available both using TCP and UDP on the same port. +func override(server string) { + net.DefaultResolver.PreferGo = true + net.DefaultResolver.Dial = func(ctx context.Context, network, _ string) (net.Conn, error) { + dialer := net.Dialer{ + // This is localhost, it is either running or not. Fail quickly if + // we can't connect. + Timeout: 1 * time.Second, + } + + switch network { + case "udp", "udp4", "udp6": + return dialer.DialContext(ctx, "udp4", server) + case "tcp", "tcp4", "tcp6": + return dialer.DialContext(ctx, "tcp4", server) + default: + panic("OverrideDNS.Dial: unknown network") + } + } +} diff --git a/framework/dns/resolver.go b/framework/dns/resolver.go new file mode 100644 index 0000000..f1393fe --- /dev/null +++ b/framework/dns/resolver.go @@ -0,0 +1,61 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package dns defines interfaces used by maddy modules to perform DNS +// lookups. +// +// Currently, there is only Resolver interface which is implemented +// by dns.DefaultResolver(). In the future, DNSSEC-enabled stub resolver +// implementation will be added here. +package dns + +import ( + "context" + "net" + "strings" +) + +// Resolver is an interface that describes DNS-related methods used by maddy. +// +// It is implemented by dns.DefaultResolver(). Methods behave the same way. +type Resolver interface { + LookupAddr(ctx context.Context, addr string) (names []string, err error) + LookupHost(ctx context.Context, host string) (addrs []string, err error) + LookupMX(ctx context.Context, name string) ([]*net.MX, error) + LookupTXT(ctx context.Context, name string) ([]string, error) + LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) +} + +// LookupAddr is a convenience wrapper for Resolver.LookupAddr. +// +// It returns the first name with trailing dot stripped. +func LookupAddr(ctx context.Context, r Resolver, ip net.IP) (string, error) { + names, err := r.LookupAddr(ctx, ip.String()) + if err != nil || len(names) == 0 { + return "", err + } + return strings.TrimRight(names[0], "."), nil +} + +func DefaultResolver() Resolver { + if overrideServ != "" && overrideServ != "system-default" { + override(overrideServ) + } + + return net.DefaultResolver +} diff --git a/framework/exterrors/dns.go b/framework/exterrors/dns.go new file mode 100644 index 0000000..fd0eeeb --- /dev/null +++ b/framework/exterrors/dns.go @@ -0,0 +1,35 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package exterrors + +import ( + "net" +) + +func UnwrapDNSErr(err error) (reason string, misc map[string]interface{}) { + dnsErr, ok := err.(*net.DNSError) + if !ok { + // Return non-nil in case the user will try to 'extend' it with its own + // values. + return "", map[string]interface{}{} + } + + // Nor server name, nor DNS name are usually useful, so exclude them. + return dnsErr.Err, map[string]interface{}{} +} diff --git a/framework/exterrors/exterrors.go b/framework/exterrors/exterrors.go new file mode 100644 index 0000000..2ed34c5 --- /dev/null +++ b/framework/exterrors/exterrors.go @@ -0,0 +1,22 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package errors defines error-handling and primitives +// used across maddy, notably to pass additional error +// information across module boundaries. +package exterrors diff --git a/framework/exterrors/fields.go b/framework/exterrors/fields.go new file mode 100644 index 0000000..8f3e8ab --- /dev/null +++ b/framework/exterrors/fields.go @@ -0,0 +1,74 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package exterrors + +type fieldsErr interface { + Fields() map[string]interface{} +} + +type unwrapper interface { + Unwrap() error +} + +type fieldsWrap struct { + err error + fields map[string]interface{} +} + +func (fw fieldsWrap) Error() string { + return fw.err.Error() +} + +func (fw fieldsWrap) Unwrap() error { + return fw.err +} + +func (fw fieldsWrap) Fields() map[string]interface{} { + return fw.fields +} + +func Fields(err error) map[string]interface{} { + fields := make(map[string]interface{}, 5) + + for err != nil { + errFields, ok := err.(fieldsErr) + if ok { + for k, v := range errFields.Fields() { + // Outer errors override fields of the inner ones. + // Not the reverse. + if fields[k] != nil { + continue + } + fields[k] = v + } + } + + unwrap, ok := err.(unwrapper) + if !ok { + break + } + err = unwrap.Unwrap() + } + + return fields +} + +func WithFields(err error, fields map[string]interface{}) error { + return fieldsWrap{err: err, fields: fields} +} diff --git a/framework/exterrors/smtp.go b/framework/exterrors/smtp.go new file mode 100644 index 0000000..01e16f4 --- /dev/null +++ b/framework/exterrors/smtp.go @@ -0,0 +1,146 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package exterrors + +import ( + "fmt" + + "github.com/emersion/go-smtp" +) + +type EnhancedCode smtp.EnhancedCode + +func (ec EnhancedCode) FormatLog() string { + return fmt.Sprintf("%d.%d.%d", ec[0], ec[1], ec[2]) +} + +// SMTPError type is a copy of emersion/go-smtp.SMTPError type +// that extends it with Fields method for logging and reporting +// in maddy. It should be used instead of the go-smtp library type for all +// errors. +type SMTPError struct { + // SMTP status code. Most of these codes are overly generic and are barely + // useful. Nonetheless, take a look at the 'Associated basic status code' + // in the SMTP Enhanced Status Codes registry (below), then check RFC 5321 + // (Section 4.3.2) and pick what you like. Stick to 451 and 554 if there are + // no useful codes. + Code int + + // Enhanced SMTP status code. If you are unsure, take a look at + // https://www.iana.org/assignments/smtp-enhanced-status-codes/smtp-enhanced-status-codes.xhtml + EnhancedCode EnhancedCode + + // Error message that should be returned to the SMTP client. + // Usually, it should be a short and generic description of the error + // that excludes any details. Especially, for checks, avoid + // mentioning the exact policy mechanism used to avoid disclosing the + // server configuration details. Don't say "DNS error during DMARC check", + // say "DNS error during policy check". Same goes for network and file I/O + // errors. ESPECIALLY, don't include any configuration variables or object + // identifiers in it. + Message string + + // If the error was generated by a message check + // this field includes module name. + CheckName string + + // If the error was generated by a delivery target + // this field includes module name. + TargetName string + + // If the error was generated by a message modifier + // this field includes module name. + ModifierName string + + // If the error was generated as a result of another + // error - this field contains the original error object. + // + // Err.Error() will be copied into the 'reason' field returned + // by the Fields method unless a different values is specified + // using the Reason field below. + Err error + + // Textual explanation of the actual error reason. Defaults to the + // Err.Error() value if Err is not nil, empty string otherwise. + Reason string + + Misc map[string]interface{} +} + +func (se *SMTPError) Unwrap() error { + return se.Err +} + +func (se *SMTPError) Fields() map[string]interface{} { + ctx := make(map[string]interface{}, len(se.Misc)+3) + for k, v := range se.Misc { + ctx[k] = v + } + ctx["smtp_code"] = se.Code + ctx["smtp_enchcode"] = se.EnhancedCode + ctx["smtp_msg"] = se.Message + if se.CheckName != "" { + ctx["check"] = se.CheckName + } + if se.TargetName != "" { + ctx["target"] = se.TargetName + } + if se.Reason != "" { + ctx["reason"] = se.Reason + } else if se.Err != nil { + ctx["reason"] = se.Err.Error() + } + return ctx +} + +// Temporary reports whether +func (se *SMTPError) Temporary() bool { + return se.Code/100 == 4 +} + +func (se *SMTPError) Error() string { + if se.Reason != "" { + return se.Reason + } + if se.Err != nil { + return se.Err.Error() + } + return se.Message +} + +// SMTPCode is a convenience function that returns one of its arguments +// depending on the result of exterrors.IsTemporary for the specified error +// object. +func SMTPCode(err error, temporaryCode, permanentCode int) int { + if IsTemporary(err) { + return temporaryCode + } + return permanentCode +} + +// SMTPEnchCode is a convenience function changes the first number of the SMTP enhanced +// status code based on the value exterrors.IsTemporary returns for the specified +// error object. +func SMTPEnchCode(err error, code EnhancedCode) EnhancedCode { + if IsTemporary(err) { + code[0] = 4 + } + code[0] = 5 + return code +} diff --git a/framework/exterrors/temporary.go b/framework/exterrors/temporary.go new file mode 100644 index 0000000..1b6db16 --- /dev/null +++ b/framework/exterrors/temporary.go @@ -0,0 +1,74 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package exterrors + +import ( + "errors" +) + +type TemporaryErr interface { + Temporary() bool +} + +// IsTemporaryOrUnspec is similar to IsTemporary except that it returns true +// if error does not have a Temporary() method. Basically, it assumes that +// errors are temporary by default compared to IsTemporary that assumes +// errors are permanent by default. +func IsTemporaryOrUnspec(err error) bool { + var temp TemporaryErr + if errors.As(err, &temp) { + return temp.Temporary() + } + return true +} + +// IsTemporary returns true whether the passed error object +// have a Temporary() method and it returns true. +func IsTemporary(err error) bool { + var temp TemporaryErr + if errors.As(err, &temp) { + return temp.Temporary() + } + return false +} + +type temporaryErr struct { + err error + temp bool +} + +func (t temporaryErr) Unwrap() error { + return t.err +} + +func (t temporaryErr) Error() string { + return t.err.Error() +} + +func (t temporaryErr) Temporary() bool { + return t.temp +} + +// WithTemporary wraps the passed error object with the implementation of the +// Temporary() method that will return the specified value. +// +// Original error value can be obtained using errors.Unwrap. +func WithTemporary(err error, temporary bool) error { + return temporaryErr{err, temporary} +} diff --git a/framework/future/future.go b/framework/future/future.go new file mode 100644 index 0000000..8e44c92 --- /dev/null +++ b/framework/future/future.go @@ -0,0 +1,105 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package future + +import ( + "context" + "runtime/debug" + "sync" + + "github.com/foxcpp/maddy/framework/log" +) + +// The Future object implements a container for (value, error) pair that "will +// be populated later" and allows multiple users to wait for it to be set. +// +// It should not be copied after first use. +type Future struct { + mu sync.RWMutex + set bool + val interface{} + err error + + notify chan struct{} +} + +func New() *Future { + return &Future{notify: make(chan struct{})} +} + +// Set sets the Future (value, error) pair. All currently blocked and future +// Get calls will return it. +func (f *Future) Set(val interface{}, err error) { + if f == nil { + panic("nil future used") + } + + f.mu.Lock() + defer f.mu.Unlock() + + if f.set { + stack := debug.Stack() + log.Println("Future.Set called multiple times", stack) + log.Println("value=", val, "err=", err) + return + } + + f.set = true + f.val = val + f.err = err + + close(f.notify) +} + +func (f *Future) Get() (interface{}, error) { + if f == nil { + panic("nil future used") + } + + return f.GetContext(context.Background()) +} + +func (f *Future) GetContext(ctx context.Context) (interface{}, error) { + if f == nil { + panic("nil future used") + } + + f.mu.RLock() + if f.set { + val := f.val + err := f.err + f.mu.RUnlock() + return val, err + } + + f.mu.RUnlock() + select { + case <-f.notify: + case <-ctx.Done(): + return nil, ctx.Err() + } + + f.mu.RLock() + defer f.mu.RUnlock() + if !f.set { + panic("future: Notification received, but value is not set") + } + + return f.val, f.err +} diff --git a/framework/future/future_test.go b/framework/future/future_test.go new file mode 100644 index 0000000..aa0d581 --- /dev/null +++ b/framework/future/future_test.go @@ -0,0 +1,75 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package future + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestFuture_SetBeforeGet(t *testing.T) { + f := New() + + f.Set(1, errors.New("1")) + val, err := f.Get() + if err.Error() != "1" { + t.Error("Wrong error:", err) + } + + if val, _ := val.(int); val != 1 { + t.Fatal("wrong val received from Get") + } +} + +func TestFuture_Wait(t *testing.T) { + f := New() + + go func() { + time.Sleep(500 * time.Millisecond) + f.Set(1, errors.New("1")) + }() + + val, err := f.Get() + if val, _ := val.(int); val != 1 { + t.Fatal("wrong val received from Get") + } + if err.Error() != "1" { + t.Error("Wrong error:", err) + } + + val, err = f.Get() + if val, _ := val.(int); val != 1 { + t.Fatal("wrong val received from Get on second try") + } + if err.Error() != "1" { + t.Error("Wrong error:", err) + } +} + +func TestFuture_WaitCtx(t *testing.T) { + f := New() + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + _, err := f.GetContext(ctx) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatal("context is not cancelled") + } +} diff --git a/framework/hooks/hooks.go b/framework/hooks/hooks.go new file mode 100644 index 0000000..7319fff --- /dev/null +++ b/framework/hooks/hooks.go @@ -0,0 +1,80 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package hooks + +import "sync" + +type Event int + +const ( + // EventShutdown is triggered when the server process is about to stop. + EventShutdown Event = iota + + // EventReload is triggered when the server process receives the SIGUSR2 + // signal (on POSIX platforms) and indicates the request to reload the + // server configuration from persistent storage. + // + // Since it is by design problematic to reload the modules configuration, + // this event only applies to secondary files such as aliases mapping and + // TLS certificates. + EventReload + + // EventLogRotate is triggered when the server process receives the SIGUSR1 + // signal (on POSIX platforms) and indicates the request to reopen used log + // files since they might have rotated. + EventLogRotate +) + +var ( + hooks = make(map[Event][]func()) + hooksLck sync.Mutex +) + +func hooksToRun(eventName Event) []func() { + hooksLck.Lock() + defer hooksLck.Unlock() + hooksEv := hooks[eventName] + if hooksEv == nil { + return nil + } + + // The slice is copied so hooks can be run without holding the lock what + // might be important since they are likely to do a lot of I/O. + hooksEvCpy := make([]func(), 0, len(hooksEv)) + hooksEvCpy = append(hooksEvCpy, hooksEv...) + + return hooksEvCpy +} + +// RunHooks runs the hooks installed for the specified eventName in the reverse +// order. +func RunHooks(eventName Event) { + hooks := hooksToRun(eventName) + for i := len(hooks) - 1; i >= 0; i-- { + hooks[i]() + } +} + +// AddHook installs the hook to be executed when certain event occurs. +func AddHook(eventName Event, f func()) { + hooksLck.Lock() + defer hooksLck.Unlock() + + hooks[eventName] = append(hooks[eventName], f) +} diff --git a/framework/log/log.go b/framework/log/log.go new file mode 100644 index 0000000..f9092a4 --- /dev/null +++ b/framework/log/log.go @@ -0,0 +1,237 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package log implements a minimalistic logging library. +package log + +import ( + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/foxcpp/maddy/framework/exterrors" + "go.uber.org/zap" +) + +// Logger is the structure that writes formatted output to the underlying +// log.Output object. +// +// Logger is stateless and can be copied freely. However, consider that +// underlying log.Output will not be copied. +// +// Each log message is prefixed with logger name. Timestamp and debug flag +// formatting is done by log.Output. +// +// No serialization is provided by Logger, its log.Output responsibility to +// ensure goroutine-safety if necessary. +type Logger struct { + Out Output + Name string + Debug bool + + // Additional fields that will be added + // to the Msg output. + Fields map[string]interface{} +} + +func (l Logger) Zap() *zap.Logger { + // TODO: Migrate to using zap natively. + return zap.New(zapLogger{L: l}) +} + +func (l Logger) Debugf(format string, val ...interface{}) { + if !l.Debug { + return + } + l.log(true, l.formatMsg(fmt.Sprintf(format, val...), nil)) +} + +func (l Logger) Debugln(val ...interface{}) { + if !l.Debug { + return + } + l.log(true, l.formatMsg(strings.TrimRight(fmt.Sprintln(val...), "\n"), nil)) +} + +func (l Logger) Printf(format string, val ...interface{}) { + l.log(false, l.formatMsg(fmt.Sprintf(format, val...), nil)) +} + +func (l Logger) Println(val ...interface{}) { + l.log(false, l.formatMsg(strings.TrimRight(fmt.Sprintln(val...), "\n"), nil)) +} + +// Msg writes an event log message in a machine-readable format (currently +// JSON). +// +// name: msg\t{"key":"value","key2":"value2"} +// +// Key-value pairs are built from fields slice which should contain key strings +// followed by corresponding values. That is, for example, []interface{"key", +// "value", "key2", "value2"}. +// +// If value in fields implements LogFormatter, it will be represented by the +// string returned by FormatLog method. Same goes for fmt.Stringer and error +// interfaces. +// +// Additionally, time.Time is written as a string in ISO 8601 format. +// time.Duration follows fmt.Stringer rule above. +func (l Logger) Msg(msg string, fields ...interface{}) { + m := make(map[string]interface{}, len(fields)/2) + fieldsToMap(fields, m) + l.log(false, l.formatMsg(msg, m)) +} + +// Error writes an event log message in a machine-readable format (currently +// JSON) containing information about the error. If err does have a Fields +// method that returns map[string]interface{}, its result will be added to the +// message. +// +// name: msg\t{"key":"value","key2":"value2"} +// +// Additionally, values from fields will be added to it, as handled by +// Logger.Msg. +// +// In the context of Error method, "msg" typically indicates the top-level +// context in which the error is *handled*. For example, if error leads to +// rejection of SMTP DATA command, msg will probably be "DATA error". +func (l Logger) Error(msg string, err error, fields ...interface{}) { + if err == nil { + return + } + + errFields := exterrors.Fields(err) + allFields := make(map[string]interface{}, len(fields)+len(errFields)+2) + for k, v := range errFields { + allFields[k] = v + } + + // If there is already a 'reason' field - use it, it probably + // provides a better explanation than error text itself. + if allFields["reason"] == nil { + allFields["reason"] = err.Error() + } + fieldsToMap(fields, allFields) + + l.log(false, l.formatMsg(msg, allFields)) +} + +func (l Logger) DebugMsg(kind string, fields ...interface{}) { + if !l.Debug { + return + } + m := make(map[string]interface{}, len(fields)/2) + fieldsToMap(fields, m) + l.log(true, l.formatMsg(kind, m)) +} + +func fieldsToMap(fields []interface{}, out map[string]interface{}) { + var lastKey string + for i, val := range fields { + if i%2 == 0 { + // Key + key, ok := val.(string) + if !ok { + // Misformatted arguments, attempt to provide useful message + // anyway. + out[fmt.Sprint("field", i)] = key + continue + } + lastKey = key + } else { + // Value + out[lastKey] = val + } + } +} + +func (l Logger) formatMsg(msg string, fields map[string]interface{}) string { + formatted := strings.Builder{} + + formatted.WriteString(msg) + formatted.WriteRune('\t') + + if len(l.Fields)+len(fields) != 0 { + if fields == nil { + fields = make(map[string]interface{}) + } + for k, v := range l.Fields { + fields[k] = v + } + if err := marshalOrderedJSON(&formatted, fields); err != nil { + // Fallback to printing the message with minimal processing. + return fmt.Sprintf("[BROKEN FORMATTING: %v] %v %+v", err, msg, fields) + } + } + + return formatted.String() +} + +type LogFormatter interface { + FormatLog() string +} + +// Write implements io.Writer, all bytes sent +// to it will be written as a separate log messages. +// No line-buffering is done. +func (l Logger) Write(s []byte) (int, error) { + l.log(false, strings.TrimRight(string(s), "\n")) + return len(s), nil +} + +// DebugWriter returns a writer that will act like Logger.Write +// but will use debug flag on messages. If Logger.Debug is false, +// Write method of returned object will be no-op. +func (l Logger) DebugWriter() io.Writer { + if !l.Debug { + return io.Discard + } + l.Debug = true + return &l +} + +func (l Logger) log(debug bool, s string) { + if l.Name != "" { + s = l.Name + ": " + s + } + + if l.Out != nil { + l.Out.Write(time.Now(), debug, s) + return + } + if DefaultLogger.Out != nil { + DefaultLogger.Out.Write(time.Now(), debug, s) + return + } + + // Logging is disabled - do nothing. +} + +// DefaultLogger is the global Logger object that is used by +// package-level logging functions. +// +// As with all other Loggers, it is not gorountine-safe on its own, +// however underlying log.Output may provide necessary serialization. +var DefaultLogger = Logger{Out: WriterOutput(os.Stderr, false)} + +func Debugf(format string, val ...interface{}) { DefaultLogger.Debugf(format, val...) } +func Debugln(val ...interface{}) { DefaultLogger.Debugln(val...) } +func Printf(format string, val ...interface{}) { DefaultLogger.Printf(format, val...) } +func Println(val ...interface{}) { DefaultLogger.Println(val...) } diff --git a/framework/log/orderedjson.go b/framework/log/orderedjson.go new file mode 100644 index 0000000..bbe01c2 --- /dev/null +++ b/framework/log/orderedjson.go @@ -0,0 +1,85 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package log + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +// To support ad-hoc parsing in a better way we want to make order of fields in +// output JSON documents determistics. Additionally, this will make them more +// human-readable when values from multiple messages are lined up to each +// other. + +type module interface { + Name() string + InstanceName() string +} + +func marshalOrderedJSON(output *strings.Builder, m map[string]interface{}) error { + order := make([]string, 0, len(m)) + for k := range m { + order = append(order, k) + } + sort.Strings(order) + + output.WriteRune('{') + for i, key := range order { + if i != 0 { + output.WriteRune(',') + } + + jsonKey, err := json.Marshal(key) + if err != nil { + return err + } + + output.Write(jsonKey) + output.WriteString(":") + + val := m[key] + switch casted := val.(type) { + case time.Time: + val = casted.Format("2006-01-02T15:04:05.000") + case time.Duration: + val = casted.String() + case LogFormatter: + val = casted.FormatLog() + case fmt.Stringer: + val = casted.String() + case module: + val = casted.Name() + "/" + casted.InstanceName() + case error: + val = casted.Error() + } + + jsonValue, err := json.Marshal(val) + if err != nil { + return err + } + output.Write(jsonValue) + } + output.WriteRune('}') + + return nil +} diff --git a/framework/log/output.go b/framework/log/output.go new file mode 100644 index 0000000..612a4bf --- /dev/null +++ b/framework/log/output.go @@ -0,0 +1,74 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package log + +import ( + "time" +) + +type Output interface { + Write(stamp time.Time, debug bool, msg string) + Close() error +} + +type multiOut struct { + outs []Output +} + +func (m multiOut) Write(stamp time.Time, debug bool, msg string) { + for _, out := range m.outs { + out.Write(stamp, debug, msg) + } +} + +func (m multiOut) Close() error { + for _, out := range m.outs { + if err := out.Close(); err != nil { + return err + } + } + return nil +} + +func MultiOutput(outputs ...Output) Output { + return multiOut{outputs} +} + +type funcOut struct { + out func(time.Time, bool, string) + close func() error +} + +func (f funcOut) Write(stamp time.Time, debug bool, msg string) { + f.out(stamp, debug, msg) +} + +func (f funcOut) Close() error { + return f.close() +} + +func FuncOutput(f func(time.Time, bool, string), close func() error) Output { + return funcOut{f, close} +} + +type NopOutput struct{} + +func (NopOutput) Write(time.Time, bool, string) {} + +func (NopOutput) Close() error { return nil } diff --git a/framework/log/syslog.go b/framework/log/syslog.go new file mode 100644 index 0000000..d608f55 --- /dev/null +++ b/framework/log/syslog.go @@ -0,0 +1,62 @@ +//go:build !windows && !plan9 +// +build !windows,!plan9 + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package log + +import ( + "fmt" + "log/syslog" + "os" + "time" +) + +type syslogOut struct { + w *syslog.Writer +} + +func (s syslogOut) Write(stamp time.Time, debug bool, msg string) { + var err error + if debug { + err = s.w.Debug(msg + "\n") + } else { + err = s.w.Info(msg + "\n") + } + + if err != nil { + fmt.Fprintf(os.Stderr, "!!! Failed to send message to syslog daemon: %v\n", err) + } +} + +func (s syslogOut) Close() error { + return s.w.Close() +} + +// SyslogOutput returns a log.Output implementation that will send +// messages to the system syslog daemon. +// +// Regular messages will be written with INFO priority, +// debug messages will be written with DEBUG priority. +// +// Returned log.Output object is goroutine-safe. +func SyslogOutput() (Output, error) { + w, err := syslog.New(syslog.LOG_MAIL|syslog.LOG_INFO, "maddy") + return syslogOut{w}, err +} diff --git a/framework/log/syslog_stub.go b/framework/log/syslog_stub.go new file mode 100644 index 0000000..bc48616 --- /dev/null +++ b/framework/log/syslog_stub.go @@ -0,0 +1,37 @@ +//go:build windows || plan9 +// +build windows plan9 + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package log + +import ( + "errors" +) + +// SyslogOutput returns a log.Output implementation that will send +// messages to the system syslog daemon. +// +// Regular messages will be written with INFO priority, +// debug messages will be written with DEBUG priority. +// +// Returned log.Output object is goroutine-safe. +func SyslogOutput() (Output, error) { + return nil, errors.New("log: syslog output is not supported on windows") +} diff --git a/framework/log/writer.go b/framework/log/writer.go new file mode 100644 index 0000000..1528c2e --- /dev/null +++ b/framework/log/writer.go @@ -0,0 +1,95 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package log + +import ( + "fmt" + "io" + "os" + "strings" + "time" +) + +type wcOutput struct { + timestamps bool + wc io.WriteCloser +} + +func (w wcOutput) Write(stamp time.Time, debug bool, msg string) { + builder := strings.Builder{} + if w.timestamps { + builder.WriteString(stamp.UTC().Format("2006-01-02T15:04:05.000Z ")) + } + if debug { + builder.WriteString("[debug] ") + } + builder.WriteString(msg) + builder.WriteRune('\n') + if _, err := io.WriteString(w.wc, builder.String()); err != nil { + fmt.Fprintf(os.Stderr, "!!! Failed to write message to log: %v\n", err) + } +} + +func (w wcOutput) Close() error { + return w.wc.Close() +} + +// WriteCloserOutput returns a log.Output implementation that +// will write formatted messages to the provided io.Writer. +// +// Closing returned log.Output object will close the underlying +// io.WriteCloser. +// +// Written messages will include timestamp formatted with millisecond +// precision and [debug] prefix for debug messages. +// If timestamps argument is false, timestamps will not be added. +// +// Returned log.Output does not provide its own serialization +// so goroutine-safety depends on the io.Writer. Most operating +// systems have atomic (read: thread-safe) implementations for +// stream I/O, so it should be safe to use WriterOutput with os.File. +func WriteCloserOutput(wc io.WriteCloser, timestamps bool) Output { + return wcOutput{timestamps, wc} +} + +type nopCloser struct { + io.Writer +} + +func (nc nopCloser) Close() error { + return nil +} + +// WriterOutput returns a log.Output implementation that +// will write formatted messages to the provided io.Writer. +// +// Closing returned log.Output object will have no effect on the +// underlying io.Writer. +// +// Written messages will include timestamp formatted with millisecond +// precision and [debug] prefix for debug messages. +// If timestamps argument is false, timestamps will not be added. +// +// Returned log.Output does not provide its own serialization +// so goroutine-safety depends on the io.Writer. Most operating +// systems have atomic (read: thread-safe) implementations for +// stream I/O, so it should be safe to use WriterOutput with os.File. +func WriterOutput(w io.Writer, timestamps bool) Output { + return wcOutput{timestamps, nopCloser{os.Stderr}} +} diff --git a/framework/log/zap.go b/framework/log/zap.go new file mode 100644 index 0000000..23821f8 --- /dev/null +++ b/framework/log/zap.go @@ -0,0 +1,57 @@ +package log + +import ( + "go.uber.org/zap/zapcore" +) + +// TODO: Migrate to using actual zapcore to improve logging performance + +type zapLogger struct { + L Logger +} + +func (l zapLogger) Enabled(level zapcore.Level) bool { + if l.L.Debug { + return true + } + return level > zapcore.DebugLevel +} + +func (l zapLogger) With(fields []zapcore.Field) zapcore.Core { + enc := zapcore.NewMapObjectEncoder() + for _, f := range fields { + f.AddTo(enc) + } + newF := make(map[string]interface{}, len(l.L.Fields)+len(enc.Fields)) + for k, v := range l.L.Fields { + newF[k] = v + } + for k, v := range enc.Fields { + newF[k] = v + } + l.L.Fields = newF + return l +} + +func (l zapLogger) Check(entry zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if l.Enabled(entry.Level) { + return ce.AddCore(entry, l) + } + return ce +} + +func (l zapLogger) Write(entry zapcore.Entry, fields []zapcore.Field) error { + enc := zapcore.NewMapObjectEncoder() + for _, f := range fields { + f.AddTo(enc) + } + if entry.LoggerName != "" { + l.L.Name += "/" + entry.LoggerName + } + l.L.log(entry.Level == zapcore.DebugLevel, l.L.formatMsg(entry.Message, enc.Fields)) + return nil +} + +func (zapLogger) Sync() error { + return nil +} diff --git a/framework/logparser/parse.go b/framework/logparser/parse.go new file mode 100644 index 0000000..a0fac77 --- /dev/null +++ b/framework/logparser/parse.go @@ -0,0 +1,124 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package parser provides utilities for parsing of structured log messsages +// generated by maddy. +package parser + +import ( + "encoding/json" + "strings" + "time" + "unicode" +) + +type ( + Msg struct { + Stamp time.Time + Debug bool + Module string + Message string + Context map[string]interface{} + } + + MalformedMsg struct { + Desc string + Err error + } +) + +const ( + ISO8601_UTC = "2006-01-02T15:04:05.000Z" +) + +func (m MalformedMsg) Error() string { + if m.Err != nil { + return "parse: " + m.Desc + ": " + m.Err.Error() + } + return "parse: " + m.Desc +} + +// Parse parses the message from the maddy log file. +// +// It assumes standard file output, including the [debug] tag and +// ISO 8601 timestamp at the start of each line. Timestamp is assumed to be in +// the UTC, as it is enforced by maddy. +// +// JSON context values are unmarshalled without any additional processing, +// notably that means that all numbers are represented as float64. +func Parse(line string) (Msg, error) { + parts := strings.Split(line, "\t") + if len(parts) != 2 { + // All messages even without a Context have a trailing \t, + // so this one is obviously malformed. + return Msg{}, MalformedMsg{Desc: "missing a tab separator"} + } + + m := Msg{ + Context: map[string]interface{}{}, + } + + // After that, the second part is the context. It can be empty, so don't fail + // if there is none. + if len(parts[1]) != 0 { + if err := json.Unmarshal([]byte(parts[1]), &m.Context); err != nil { + return Msg{}, MalformedMsg{Desc: "context unmarshal", Err: err} + } + } + + // Okay, the first one might contain the timestamp at start. + // Cut it away. + msgParts := strings.SplitN(parts[0], " ", 2) + if len(msgParts) == 1 { + return Msg{}, MalformedMsg{Desc: "missing a timestamp"} + } + + var err error + m.Stamp, err = time.ParseInLocation(ISO8601_UTC, msgParts[0], time.UTC) + if err != nil { + return Msg{}, MalformedMsg{Desc: "timestamp parse", Err: err} + } + + msgText := msgParts[1] + if strings.HasPrefix(msgText, "[debug] ") { + msgText = strings.TrimPrefix(msgText, "[debug] ") + m.Debug = true + } + + moduleText := strings.SplitN(msgText, ": ", 2) + if len(moduleText) == 1 { + // No module prefix, that's fine. + m.Message = msgText + return m, nil + } + + for _, ch := range moduleText[0] { + switch { + case unicode.IsDigit(ch), unicode.IsLetter(ch), ch == '/': + default: + // This is not a module prefix, don't treat it as such. + m.Message = msgText + return m, nil + } + } + + m.Module = moduleText[0] + m.Message = moduleText[1] + + return m, nil +} diff --git a/framework/logparser/parse_test.go b/framework/logparser/parse_test.go new file mode 100644 index 0000000..6361541 --- /dev/null +++ b/framework/logparser/parse_test.go @@ -0,0 +1,113 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package parser + +import ( + "reflect" + "testing" + "time" +) + +func TestParse(t *testing.T) { + test := func(line string, msg Msg, errDesc string) { + t.Helper() + + parsed, err := Parse(line) + if errDesc != "" { + if err == nil { + t.Errorf("Expected an error, got none") + return + } + if err.(MalformedMsg).Desc != errDesc { + t.Errorf("Wrong error desc returned: %v", err.(MalformedMsg).Desc) + return + } + } + if errDesc == "" && err != nil { + t.Errorf("Unexpected error: %v", err) + return + } + + if !reflect.DeepEqual(parsed, msg) { + t.Errorf("Wrong Parse result,\n got %#+v\n want %#+v", parsed, msg) + } + } + + test("2006-01-02T15:04:05.000Z module: hello\t", Msg{ + Stamp: time.Date(2006, time.January, 2, 15, 4, 5, 0, time.UTC), + Module: "module", + Message: "hello", + Context: map[string]interface{}{}, + }, "") + test("2006-01-02T15:04:05.000Z module: hello: whatever\t", Msg{ + Stamp: time.Date(2006, time.January, 2, 15, 4, 5, 0, time.UTC), + Module: "module", + Message: "hello: whatever", + Context: map[string]interface{}{}, + }, "") + test("2006-01-02T15:04:05.000Z module: hello: whatever\t{\"a\":1}", Msg{ + Stamp: time.Date(2006, time.January, 2, 15, 4, 5, 0, time.UTC), + Module: "module", + Message: "hello: whatever", + Context: map[string]interface{}{ + "a": float64(1), + }, + }, "") + test("2006-01-02T15:04:05.000Z module: hello: whatever\t{\"a\":1,\"b\":\"bbb\"}", Msg{ + Stamp: time.Date(2006, time.January, 2, 15, 4, 5, 0, time.UTC), + Module: "module", + Message: "hello: whatever", + Context: map[string]interface{}{ + "a": float64(1), + "b": "bbb", + }, + }, "") + test("2006-01-02T15:04:05.000Z [debug] module: hello: whatever\t{\"a\":1,\"b\":\"bbb\"}", Msg{ + Stamp: time.Date(2006, time.January, 2, 15, 4, 5, 0, time.UTC), + Debug: true, + Module: "module", + Message: "hello: whatever", + Context: map[string]interface{}{ + "a": float64(1), + "b": "bbb", + }, + }, "") + test("2006-01-02T15:04:05.000Z [debug] oink oink: hello: whatever\t{\"a\":1,\"b\":\"bbb\"}", Msg{ + Stamp: time.Date(2006, time.January, 2, 15, 4, 5, 0, time.UTC), + Debug: true, + Message: "oink oink: hello: whatever", + Context: map[string]interface{}{ + "a": float64(1), + "b": "bbb", + }, + }, "") + test("2006-01-02T15:04:05.000Z [debug] whatever\t{\"a\":1,\"b\":\"bbb\"}", Msg{ + Stamp: time.Date(2006, time.January, 2, 15, 4, 5, 0, time.UTC), + Debug: true, + Message: "whatever", + Context: map[string]interface{}{ + "a": float64(1), + "b": "bbb", + }, + }, "") + test("module: hello\t", Msg{}, "timestamp parse") + test("hello\t", Msg{}, "missing a timestamp") + test("2006-01-02T15:04:05.000Z module: hello", Msg{}, "missing a tab separator") + test("2006-01-02T15:04:05.000Z [BROKEN FORMATTING: json: wtf lol omg]: hello map[stringasdasd]", Msg{}, "missing a tab separator") +} diff --git a/framework/module/auth.go b/framework/module/auth.go new file mode 100644 index 0000000..6e8d089 --- /dev/null +++ b/framework/module/auth.go @@ -0,0 +1,44 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import "errors" + +// ErrUnknownCredentials should be returned by auth. provider if supplied +// credentials are valid for it but are not recognized (e.g. not found in +// used DB). +var ErrUnknownCredentials = errors.New("unknown credentials") + +// PlainAuth is the interface implemented by modules providing authentication using +// username:password pairs. +// +// Modules implementing this interface should be registered with "auth." prefix in name. +type PlainAuth interface { + AuthPlain(username, password string) error +} + +// PlainUserDB is a local credentials store that can be managed using maddy command +// utility. +type PlainUserDB interface { + PlainAuth + ListUsers() ([]string, error) + CreateUser(username, password string) error + SetUserPassword(username, password string) error + DeleteUser(username string) error +} diff --git a/framework/module/blob_store.go b/framework/module/blob_store.go new file mode 100644 index 0000000..426a854 --- /dev/null +++ b/framework/module/blob_store.go @@ -0,0 +1,46 @@ +package module + +import ( + "context" + "errors" + "io" +) + +type Blob interface { + Sync() error + io.Writer + io.Closer +} + +var ErrNoSuchBlob = errors.New("blob_store: no such object") + +const UnknownBlobSize int64 = -1 + +// BlobStore is the interface used by modules providing large binary object +// storage. +type BlobStore interface { + // Create creates a new blob for writing. + // + // Sync will be called on the returned Blob object after -all- data has + // been successfully written. + // + // Close without Sync can be assumed to happen due to an unrelated error + // and stored data can be discarded. + // + // blobSize indicates the exact amount of bytes that will be written + // If -1 is passed - it is unknown and implementation will not make + // any assumptions about the blob size. Error can be returned by any + // Blob method if more than than blobSize bytes get written. + // + // Passed context will cover the entire blob write operation. + Create(ctx context.Context, key string, blobSize int64) (Blob, error) + + // Open returns the reader for the object specified by + // passed key. + // + // If no such object exists - ErrNoSuchBlob is returned. + Open(ctx context.Context, key string) (io.ReadCloser, error) + + // Delete removes a set of keys from store. Non-existent keys are ignored. + Delete(ctx context.Context, keys []string) error +} diff --git a/framework/module/check.go b/framework/module/check.go new file mode 100644 index 0000000..4802f6e --- /dev/null +++ b/framework/module/check.go @@ -0,0 +1,113 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/foxcpp/maddy/framework/buffer" +) + +// Check is the module interface that is meant for read-only (with the +// exception of the message header modifications) (meta-)data checking. +// +// Modules implementing this interface should be registered with "check." +// prefix in name. +type Check interface { + // CheckStateForMsg initializes the "internal" check state required for + // processing of the new message. + // + // NOTE: Returned CheckState object must be hashable (usable as a map key). + // This is used to deduplicate Check* calls, the easiest way to achieve + // this is to have CheckState as a pointer to some struct, all pointers + // are hashable. + CheckStateForMsg(ctx context.Context, msgMeta *MsgMetadata) (CheckState, error) +} + +// EarlyCheck is an optional module interface that can be implemented +// by module implementing Check. +// +// It is used as an optimization to reject obviously malicious connections +// before allocating resources for SMTP session. +// +// The Status of this check is accept (no error) or reject (error) only, no +// advanced handling is available (such as 'quarantine' action and headers +// prepending). +// +// If it s necessary to defer or affect further message processing +// without outright killing the session, ConnState.ModData can be +// used to store necessary information. +// +// It may be called multiple times for the same connection if TLS is negotiated +// via STARTTLS. In this case, no state will be passed between before-TLS +// context to the TLS one. +type EarlyCheck interface { + CheckConnection(ctx context.Context, state *ConnState) error +} + +type CheckState interface { + // CheckConnection is executed once when client sends a new message. + CheckConnection(ctx context.Context) CheckResult + + // CheckSender is executed once when client sends the message sender + // information (e.g. on the MAIL FROM command). + CheckSender(ctx context.Context, mailFrom string) CheckResult + + // CheckRcpt is executed for each recipient when its address is received + // from the client (e.g. on the RCPT TO command). + CheckRcpt(ctx context.Context, rcptTo string) CheckResult + + // CheckBody is executed once after the message body is received and + // buffered in memory or on disk. + // + // Check code should use passed mutex when working with the message header. + // Body can be read without locking it since it is read-only. + CheckBody(ctx context.Context, header textproto.Header, body buffer.Buffer) CheckResult + + // Close is called after the message processing ends, even if any of the + // Check* functions return an error. + Close() error +} + +type CheckResult struct { + // Reason is the error that is reported to the message source + // if check decided that the message should be rejected. + Reason error + + // Reject is the flag that specifies that the message + // should be rejected. + Reject bool + + // Quarantine is the flag that specifies that the message + // is considered "possibly malicious" and should be + // put into Junk mailbox. + // + // This value is copied into MsgMetadata by the msgpipeline. + Quarantine bool + + // AuthResult is the information that is supposed to + // be included in Authentication-Results header. + AuthResult []authres.Result + + // Header is the header fields that should be + // added to the header after all checks. + Header textproto.Header +} diff --git a/framework/module/delivery_target.go b/framework/module/delivery_target.go new file mode 100644 index 0000000..9a7c1e0 --- /dev/null +++ b/framework/module/delivery_target.go @@ -0,0 +1,93 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" +) + +// DeliveryTarget interface represents abstract storage for the message data +// (typically persistent) or other kind of component that can be used as a +// final destination for the message. +// +// Modules implementing this interface should be registered with "target." +// prefix in name. +type DeliveryTarget interface { + // Start starts the delivery of a new message. + // + // The domain part of the MAIL FROM address is assumed to be U-labels with + // NFC normalization and case-folding applied. The message source should + // ensure that by calling address.CleanDomain if necessary. + Start(ctx context.Context, msgMeta *MsgMetadata, mailFrom string) (Delivery, error) +} + +type Delivery interface { + // AddRcpt adds the target address for the message. + // + // The domain part of the address is assumed to be U-labels with NFC normalization + // and case-folding applied. The message source should ensure that by + // calling address.CleanDomain if necessary. + // + // Implementation should assume that no case-folding or deduplication was + // done by caller code. Its implementation responsibility to do so if it is + // necessary. It is not recommended to reject duplicated recipients, + // however. They should be silently ignored. + // + // Implementation should do as much checks as possible here and reject + // recipients that can't be used. Note: MsgMetadata object passed to Start + // contains BodyLength field. If it is non-zero, it can be used to check + // storage quota for the user before Body. + AddRcpt(ctx context.Context, rcptTo string, opts smtp.RcptOptions) error + + // Body sets the body and header contents for the message. + // If this method fails, message is assumed to be undeliverable + // to all recipients. + // + // Implementation should avoid doing any persistent changes to the + // underlying storage until Commit is called. If that is not possible, + // Abort should (attempt to) rollback any such changes. + // + // If Body can't be implemented without per-recipient failures, + // then delivery object should also implement PartialDelivery interface + // for use by message sources that are able to make sense of per-recipient + // errors. + // + // Here is the example of possible implementation for maildir-based + // storage: + // Calling Body creates a file in tmp/ directory. + // Commit moves the created file to new/ directory. + // Abort removes the created file. + Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error + + // Abort cancels message delivery. + // + // All changes made to the underlying storage should be aborted at this + // point, if possible. + Abort(ctx context.Context) error + + // Commit completes message delivery. + // + // It generally should never fail, since failures here jeopardize + // atomicity of the delivery if multiple targets are used. + Commit(ctx context.Context) error +} diff --git a/framework/module/dummy.go b/framework/module/dummy.go new file mode 100644 index 0000000..0930d4d --- /dev/null +++ b/framework/module/dummy.go @@ -0,0 +1,87 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" +) + +// Dummy is a struct that implements PlainAuth and DeliveryTarget +// interfaces but does nothing. Useful for testing. +// +// It is always registered under the 'dummy' name and can be used in both tests +// and the actual server code (but the latter is kinda pointless). +type Dummy struct{ instName string } + +func (d *Dummy) AuthPlain(username, _ string) error { + return nil +} + +func (d *Dummy) Lookup(_ context.Context, _ string) (string, bool, error) { + return "", false, nil +} + +func (d *Dummy) LookupMulti(_ context.Context, _ string) ([]string, error) { + return []string{""}, nil +} + +func (d *Dummy) Name() string { + return "dummy" +} + +func (d *Dummy) InstanceName() string { + return d.instName +} + +func (d *Dummy) Init(_ *config.Map) error { + return nil +} + +func (d *Dummy) Start(ctx context.Context, msgMeta *MsgMetadata, mailFrom string) (Delivery, error) { + return dummyDelivery{}, nil +} + +type dummyDelivery struct{} + +func (dd dummyDelivery) AddRcpt(ctx context.Context, rcptTo string, opts smtp.RcptOptions) error { + return nil +} + +func (dd dummyDelivery) Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error { + return nil +} + +func (dd dummyDelivery) Abort(ctx context.Context) error { + return nil +} + +func (dd dummyDelivery) Commit(ctx context.Context) error { + return nil +} + +func init() { + Register("dummy", func(_, instName string, _, _ []string) (Module, error) { + return &Dummy{instName: instName}, nil + }) +} diff --git a/framework/module/imap_filter.go b/framework/module/imap_filter.go new file mode 100644 index 0000000..6b0fd46 --- /dev/null +++ b/framework/module/imap_filter.go @@ -0,0 +1,43 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" +) + +// IMAPFilter is interface used by modules that want to modify IMAP-specific message +// attributes on delivery. +// +// Modules implementing this interface should be registered with namespace prefix +// "imap.filter". +type IMAPFilter interface { + // IMAPFilter is called when message is about to be stored in IMAP-compatible + // storage. It is called only for messages delivered over SMTP, hdr and body + // contain the message exactly how it will be stored. + // + // Filter can change the target directory by returning non-empty folder value. + // Additionally it can add additional IMAP flags to the message by returning + // them. + // + // Errors returned by IMAPFilter will be just logged and will not cause delivery + // to fail. + IMAPFilter(accountName string, rcptTo string, meta *MsgMetadata, hdr textproto.Header, body buffer.Buffer) (folder string, flags []string, err error) +} diff --git a/framework/module/instances.go b/framework/module/instances.go new file mode 100644 index 0000000..aa6f148 --- /dev/null +++ b/framework/module/instances.go @@ -0,0 +1,105 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "fmt" + "io" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/log" +) + +var ( + instances = make(map[string]struct { + mod Module + cfg *config.Map + }) + aliases = make(map[string]string) + + Initialized = make(map[string]bool) +) + +// RegisterInstance adds module instance to the global registry. +// +// Instance name must be unique. Second RegisterInstance with same instance +// name will replace previous. +func RegisterInstance(inst Module, cfg *config.Map) { + instances[inst.InstanceName()] = struct { + mod Module + cfg *config.Map + }{inst, cfg} +} + +// RegisterAlias creates an association between a certain name and instance name. +// +// After RegisterAlias, module.GetInstance(aliasName) will return the same +// result as module.GetInstance(instName). +func RegisterAlias(aliasName, instName string) { + aliases[aliasName] = instName +} + +func HasInstance(name string) bool { + aliasedName := aliases[name] + if aliasedName != "" { + name = aliasedName + } + + _, ok := instances[name] + return ok +} + +// GetInstance returns module instance from global registry, initializing it if +// necessary. +// +// Error is returned if module initialization fails or module instance does not +// exists. +func GetInstance(name string) (Module, error) { + aliasedName := aliases[name] + if aliasedName != "" { + name = aliasedName + } + + mod, ok := instances[name] + if !ok { + return nil, fmt.Errorf("unknown config block: %s", name) + } + + // Break circular dependencies. + if Initialized[name] { + return mod.mod, nil + } + + Initialized[name] = true + if err := mod.mod.Init(mod.cfg); err != nil { + return mod.mod, err + } + + if closer, ok := mod.mod.(io.Closer); ok { + hooks.AddHook(hooks.EventShutdown, func() { + log.Debugf("close %s (%s)", mod.mod.Name(), mod.mod.InstanceName()) + if err := closer.Close(); err != nil { + log.Printf("module %s (%s) close failed: %v", mod.mod.Name(), mod.mod.InstanceName(), err) + } + }) + } + + return mod.mod, nil +} diff --git a/framework/module/modifier.go b/framework/module/modifier.go new file mode 100644 index 0000000..2a5b10f --- /dev/null +++ b/framework/module/modifier.go @@ -0,0 +1,83 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" +) + +// Modifier is the module interface for modules that can mutate the +// processed message or its meta-data. +// +// Currently, the message body can't be mutated for efficiency and +// correctness reasons: It would require "rebuffering" (see buffer.Buffer doc), +// can invalidate assertions made on the body contents before modification and +// will break DKIM signatures. +// +// Only message header can be modified. Furthermore, it is highly discouraged for +// modifiers to remove or change existing fields to prevent issues outlined +// above. +// +// Calls on ModifierState are always strictly ordered. +// RewriteRcpt is newer called before RewriteSender and RewriteBody is never called +// before RewriteRcpts. This allows modificator code to save values +// passed to previous calls for use in later operations. +// +// Modules implementing this interface should be registered with "modify." prefix in name. +type Modifier interface { + // ModStateForMsg initializes modifier "internal" state + // required for processing of the message. + ModStateForMsg(ctx context.Context, msgMeta *MsgMetadata) (ModifierState, error) +} + +type ModifierState interface { + // RewriteSender allows modifier to replace MAIL FROM value. + // If no changes are required, this method returns its + // argument, otherwise it returns a new value. + // + // Note that per-source/per-destination modifiers are executed + // after routing decision is made so changed value will have no + // effect on it. + // + // Also note that MsgMeta.OriginalFrom will still contain the original value + // for purposes of tracing. It should not be modified by this method. + RewriteSender(ctx context.Context, mailFrom string) (string, error) + + // RewriteRcpt replaces RCPT TO value. + // If no changed are required, this method returns its argument as slice, + // otherwise it returns a slice with 1 or more new values. + // + // MsgPipeline will take of populating MsgMeta.OriginalRcpts. RewriteRcpt + // doesn't do it. + RewriteRcpt(ctx context.Context, rcptTo string) ([]string, error) + + // RewriteBody modifies passed Header argument and may optionally + // inspect the passed body buffer to make a decision on new header field values. + // + // There is no way to modify the body and RewriteBody should avoid + // removing existing header fields and changing their values. + RewriteBody(ctx context.Context, h *textproto.Header, body buffer.Buffer) error + + // Close is called after the message processing ends, even if any of the + // Rewrite* functions return an error. + Close() error +} diff --git a/framework/module/module.go b/framework/module/module.go new file mode 100644 index 0000000..2dbb45e --- /dev/null +++ b/framework/module/module.go @@ -0,0 +1,90 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package module contains modules registry and interfaces implemented +// by modules. +// +// Interfaces are placed here to prevent circular dependencies. +// +// Each interface required by maddy for operation is provided by some object +// called "module". This includes authentication, storage backends, DKIM, +// email filters, etc. Each module may serve multiple functions. I.e. it can +// be IMAP storage backend, SMTP downstream and authentication provider at the +// same moment. +// +// Each module gets its own unique name (sql for go-imap-sql, proxy for +// proxy module, local for local delivery perhaps, etc). Each module instance +// also can have its own unique name can be used to refer to it in +// configuration. +package module + +import ( + "github.com/foxcpp/maddy/framework/config" +) + +// Module is the interface implemented by all maddy module instances. +// +// It defines basic methods used to identify instances. +// +// Additionally, module can implement io.Closer if it needs to perform clean-up +// on shutdown. If module starts long-lived goroutines - they should be stopped +// *before* Close method returns to ensure graceful shutdown. +type Module interface { + // Init performs actual initialization of the module. + // + // It is not done in FuncNewModule so all module instances are + // registered at time of initialization, thus initialization does not + // depends on ordering of configuration blocks and modules can reference + // each other without any problems. + // + // Module can use passed config.Map to read its configuration variables. + Init(*config.Map) error + + // Name method reports module name. + // + // It is used to reference module in the configuration and in logs. + Name() string + + // InstanceName method reports unique name of this module instance or empty + // string if module instance is unnamed. + InstanceName() string +} + +// FuncNewModule is function that creates new instance of module with specified name. +// +// Module.InstanceName() of the returned module object should return instName. +// aliases slice contains other names that can be used to reference created +// module instance. +// +// If module is defined inline, instName will be empty and all values +// specified after module name in configuration will be in inlineArgs. +type FuncNewModule func(modName, instName string, aliases, inlineArgs []string) (Module, error) + +// FuncNewEndpoint is a function that creates new instance of endpoint +// module. +// +// Compared to regular modules, endpoint module instances are: +// - Not registered in the global registry. +// - Can't be defined inline. +// - Don't have an unique name +// - All config arguments are always passed as an 'addrs' slice and not used as +// names. +// +// As a consequence of having no per-instance name, InstanceName of the module +// object always returns the same value as Name. +type FuncNewEndpoint func(modName string, addrs []string) (Module, error) diff --git a/framework/module/module_specific_data.go b/framework/module/module_specific_data.go new file mode 100644 index 0000000..4155a61 --- /dev/null +++ b/framework/module/module_specific_data.go @@ -0,0 +1,63 @@ +package module + +import ( + "encoding/json" + "fmt" + "sync" +) + +// ModSpecificData is a container that allows modules to attach +// additional context data to framework objects such as SMTP connections +// without conflicting with each other and ensuring each module +// gets its own namespace. +// +// It must not be used to store stateful objects that may need +// a specific cleanup routine as ModSpecificData does not provide +// any lifetime management. +// +// Stored data must be serializable to JSON for state persistence +// e.g. when message is stored in a on-disk queue. +type ModSpecificData struct { + modDataLck sync.RWMutex + modData map[string]interface{} +} + +func (msd *ModSpecificData) modKey(m Module, perInstance bool) string { + if !perInstance { + return m.Name() + } + instName := m.InstanceName() + if instName == "" { + instName = fmt.Sprintf("%x", m) + } + return m.Name() + "/" + instName +} + +func (msd *ModSpecificData) MarshalJSON() ([]byte, error) { + msd.modDataLck.RLock() + defer msd.modDataLck.RUnlock() + return json.Marshal(msd.modData) +} + +func (msd *ModSpecificData) UnmarshalJSON(b []byte) error { + msd.modDataLck.Lock() + defer msd.modDataLck.Unlock() + return json.Unmarshal(b, &msd.modData) +} + +func (msd *ModSpecificData) Set(m Module, perInstance bool, value interface{}) { + key := msd.modKey(m, perInstance) + msd.modDataLck.Lock() + defer msd.modDataLck.Unlock() + if msd.modData == nil { + msd.modData = make(map[string]interface{}) + } + msd.modData[key] = value +} + +func (msd *ModSpecificData) Get(m Module, perInstance bool) interface{} { + key := msd.modKey(m, perInstance) + msd.modDataLck.RLock() + defer msd.modDataLck.RUnlock() + return msd.modData[key] +} diff --git a/framework/module/msgmetadata.go b/framework/module/msgmetadata.go new file mode 100644 index 0000000..bfbe634 --- /dev/null +++ b/framework/module/msgmetadata.go @@ -0,0 +1,158 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "crypto/rand" + "crypto/tls" + "encoding/hex" + "io" + "net" + + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/future" +) + +// ConnState structure holds the state information of the protocol used to +// accept this message. +type ConnState struct { + // IANA name (ESMTP, ESMTPS, etc) of the protocol message was received + // over. If the message was generated locally, this field is empty. + Proto string + + // Information about the SMTP connection, including HELO hostname and + // source IP. Valid only if Proto refers the SMTP protocol or its variant + // (e.g. LMTP). + Hostname string + LocalAddr net.Addr + RemoteAddr net.Addr + TLS tls.ConnectionState + + // The RDNSName field contains the result of Reverse DNS lookup on the + // client IP. + // + // The underlying type is the string or untyped nil value. It is the + // message source responsibility to populate this field. + // + // Valid values of this field consumers need to be aware of: + // RDNSName = nil + // The reverse DNS lookup is not applicable for that message source. + // Typically the case for messages generated locally. + // RDNSName != nil, but Get returns nil + // The reverse DNS lookup was attempted, but resulted in an error. + // Consumers should assume that the PTR record doesn't exist. + RDNSName *future.Future + + // If the client successfully authenticated using a username/password pair. + // This field contains the username. + AuthUser string + + // If the client successfully authenticated using a username/password pair. + // This field should be cleaned if the ConnState object is serialized + AuthPassword string + + ModData ModSpecificData +} + +// MsgMetadata structure contains all information about the origin of +// the message and all associated flags indicating how it should be handled +// by components. +// +// All fields should be considered read-only except when otherwise is noted. +// Module instances should avoid keeping reference to the instance passed to it +// and copy the structure using DeepCopy method instead. +// +// Compatibility with older values should be considered when changing this +// structure since it is serialized to the disk by the queue module using +// JSON. Modules should correctly handle missing or invalid values. +type MsgMetadata struct { + // Unique identifier for this message. Randomly generated by the + // message source module. + ID string + + // Original message sender address as it was received by the message source. + // + // Note that this field is meant for use for tracing purposes. + // All routing and other decisions should be made based on the sender address + // passed separately (for example, mailFrom argument for CheckSender function) + // Note that addresses may contain unescaped Unicode characters. + OriginalFrom string + + // If set - no SrcHostname and SrcAddr will be added to Received + // header. These fields are still written to the server log. + DontTraceSender bool + + // Quarantine is a message flag that is should be set if message is + // considered "suspicious" and should be put into "Junk" folder + // in the storage. + // + // This field should not be modified by the checks that verify + // the message. It is set only by the message pipeline. + Quarantine bool + + // OriginalRcpts contains the mapping from the final recipient to the + // recipient that was presented by the client. + // + // MsgPipeline will update that field when recipient modifiers + // are executed. + // + // It should be used when reporting information back to client (via DSN, + // for example) to prevent disclosing information about aliases + // which is usually unwanted. + OriginalRcpts map[string]string + + // SMTPOpts contains the SMTP MAIL FROM command arguments, if the message + // was accepted over SMTP or SMTP-like protocol (such as LMTP). + // + // Note that the Size field should not be used as source of information about + // the body size. Especially since it counts the header too whereas + // Buffer.Len does not. + SMTPOpts smtp.MailOptions + + // Conn contains the information about the underlying protocol connection + // that was used to accept this message. The referenced instance may be shared + // between multiple messages. + // + // It can be nil for locally generated messages. + Conn *ConnState + + // This is set by endpoint/smtp to indicate that body contains "TLS-Required: No" + // header. It is only meaningful if server has seen the body at least once + // (e.g. the message was passed via queue). + TLSRequireOverride bool +} + +// DeepCopy creates a copy of the MsgMetadata structure, also +// copying contents of the maps and slices. +// +// There are a few exceptions, however: +// - SrcAddr is not copied and copy field references original value. +func (msgMeta *MsgMetadata) DeepCopy() *MsgMetadata { + cpy := *msgMeta + // There is no good way to copy net.Addr, but it should not be + // modified by anything anyway so we are safe. + return &cpy +} + +// GenerateMsgID generates a string usable as MsgID field in module.MsgMeta. +func GenerateMsgID() (string, error) { + rawID := make([]byte, 4) + _, err := io.ReadFull(rand.Reader, rawID) + return hex.EncodeToString(rawID), err +} diff --git a/framework/module/mxauth.go b/framework/module/mxauth.go new file mode 100644 index 0000000..5226fb0 --- /dev/null +++ b/framework/module/mxauth.go @@ -0,0 +1,156 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "context" + "crypto/tls" +) + +const ( + AuthDisabled = "off" + AuthMTASTS = "mtasts" + AuthDNSSEC = "dnssec" + AuthCommonDomain = "common_domain" +) + +type ( + TLSLevel int + MXLevel int +) + +const ( + TLSNone TLSLevel = iota + TLSEncrypted + TLSAuthenticated +) + +const ( + MXNone MXLevel = iota + MX_MTASTS + MX_DNSSEC +) + +func (l TLSLevel) String() string { + switch l { + case TLSNone: + return "none" + case TLSEncrypted: + return "encrypted" + case TLSAuthenticated: + return "authenticated" + } + return "???" +} + +func (l MXLevel) String() string { + switch l { + case MXNone: + return "none" + case MX_MTASTS: + return "mtasts" + case MX_DNSSEC: + return "dnssec" + } + return "???" +} + +type ( + // MXAuthPolicy is an object that provides security check for outbound connections. + // It can do one of the following: + // + // - Check effective TLS level or MX level against some configured or + // discovered value. + // E.g. local policy. + // + // - Raise the security level if certain condition about used MX or + // connection is met. + // E.g. DANE MXAuthPolicy raises TLS level to Authenticated if a matching + // TLSA record is discovered. + // + // - Reject the connection if certain condition about used MX or + // connection is _not_ met. + // E.g. An enforced MTA-STS MXAuthPolicy rejects MX records not matching it. + // + // It is not recommended to mix different types of behavior described above + // in the same implementation. + // Specifically, the first type is used mostly for local policies and is not + // really practical. + // + // Modules implementing this interface should be registered with "mx_auth." + // prefix in name. + MXAuthPolicy interface { + Start(*MsgMetadata) DeliveryMXAuthPolicy + + // Weight is an integer in range 0-1000 that represents relative + // ordering of policy application. + Weight() int + } + + // DeliveryMXAuthPolicy is an interface of per-delivery object that estabilishes + // and verifies required and effective security for MX records and TLS + // connections. + DeliveryMXAuthPolicy interface { + // PrepareDomain is called before DNS MX lookup and may asynchronously + // start additional lookups necessary for policy application in CheckMX + // or CheckConn. + // + // If there any errors - they should be deferred to the CheckMX or + // CheckConn call. + PrepareDomain(ctx context.Context, domain string) + + // PrepareConn is called before connection and may asynchronously + // start additional lookups necessary for policy application in + // CheckConn. + // + // If there are any errors - they should be deferred to the CheckConn + // call. + PrepareConn(ctx context.Context, mx string) + + // CheckMX is called to check whether the policy permits to use a MX. + // + // mxLevel contains the MX security level estabilished by checks + // executed before. + // + // domain is passed to the CheckMX to allow simpler implementation + // of stateless policy objects. + // + // dnssec is true if the MX lookup was performed using DNSSEC-enabled + // resolver and the zone is signed and its signature is valid. + CheckMX(ctx context.Context, mxLevel MXLevel, domain, mx string, dnssec bool) (MXLevel, error) + + // CheckConn is called to check whether the policy permits to use this + // connection. + // + // tlsLevel and mxLevel contain the TLS security level estabilished by + // checks executed before. + // + // domain is passed to the CheckConn to allow simpler implementation + // of stateless policy objects. + // + // If tlsState.HandshakeCompleted is false, TLS is not used. If + // tlsState.VerifiedChains is nil, InsecureSkipVerify was used (no + // ServerName or PKI check was done). + CheckConn(ctx context.Context, mxLevel MXLevel, tlsLevel TLSLevel, domain, mx string, tlsState tls.ConnectionState) (TLSLevel, error) + + // Reset cleans the internal object state for use with another message. + // newMsg may be nil if object is not needed anymore. + Reset(newMsg *MsgMetadata) + } +) diff --git a/framework/module/partial_delivery.go b/framework/module/partial_delivery.go new file mode 100644 index 0000000..beeb46e --- /dev/null +++ b/framework/module/partial_delivery.go @@ -0,0 +1,58 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" +) + +// StatusCollector is an object that is passed by message source +// that is interested in intermediate status reports about partial +// delivery failures. +type StatusCollector interface { + // SetStatus sets the error associated with the recipient. + // + // rcptTo should match exactly the value that was passed to the + // AddRcpt, i.e. if any translations was made by the target, + // they should not affect the rcptTo argument here. + // + // It should not be called multiple times for the same + // value of rcptTo. It also should not be called + // after BodyNonAtomic returns. + // + // SetStatus is goroutine-safe. Implementations + // provide necessary serialization. + SetStatus(rcptTo string, err error) +} + +// PartialDelivery is an optional interface that may be implemented +// by the object returned by DeliveryTarget.Start. See PartialDelivery.BodyNonAtomic +// documentation for details. +type PartialDelivery interface { + // BodyNonAtomic is similar to Body method of the regular Delivery interface + // with the except that it allows target to reject the body only for some + // recipients by setting statuses using passed collector object. + // + // This interface is preferred by the LMTP endpoint and queue implementation + // to ensure correct handling of partial failures. + BodyNonAtomic(ctx context.Context, c StatusCollector, header textproto.Header, body buffer.Buffer) +} diff --git a/framework/module/registry.go b/framework/module/registry.go new file mode 100644 index 0000000..c52210f --- /dev/null +++ b/framework/module/registry.go @@ -0,0 +1,103 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "sync" + + "github.com/foxcpp/maddy/framework/log" +) + +var ( + // NoRun makes sure modules do not start any bacground tests. + // + // If it set - modules should not perform any actual work and should stop + // once the configuration is read and verified to be correct. + // TODO: Replace it with separation of Init and Run at interface level. + NoRun = false + + modules = make(map[string]FuncNewModule) + endpoints = make(map[string]FuncNewEndpoint) + modulesLock sync.RWMutex +) + +// Register adds module factory function to global registry. +// +// name must be unique. Register will panic if module with specified name +// already exists in registry. +// +// You probably want to call this function from func init() of module package. +func Register(name string, factory FuncNewModule) { + modulesLock.Lock() + defer modulesLock.Unlock() + + if _, ok := modules[name]; ok { + panic("Register: module with specified name is already registered: " + name) + } + + modules[name] = factory +} + +// RegisterDeprecated adds module factory function to global registry. +// +// It prints warning to the log about name being deprecated and suggests using +// a new name. +func RegisterDeprecated(name, newName string, factory FuncNewModule) { + Register(name, func(modName, instName string, aliases, inlineArgs []string) (Module, error) { + log.Printf("module initialized via deprecated name %s, %s should be used instead; deprecated name may be removed in the next version", name, newName) + return factory(modName, instName, aliases, inlineArgs) + }) +} + +// Get returns module from global registry. +// +// This function does not return endpoint-type modules, use GetEndpoint for +// that. +// Nil is returned if no module with specified name is registered. +func Get(name string) FuncNewModule { + modulesLock.RLock() + defer modulesLock.RUnlock() + + return modules[name] +} + +// GetEndpoints returns an endpoint module from global registry. +// +// Nil is returned if no module with specified name is registered. +func GetEndpoint(name string) FuncNewEndpoint { + modulesLock.RLock() + defer modulesLock.RUnlock() + + return endpoints[name] +} + +// RegisterEndpoint registers an endpoint module. +// +// See FuncNewEndpoint for information about +// differences of endpoint modules from regular modules. +func RegisterEndpoint(name string, factory FuncNewEndpoint) { + modulesLock.Lock() + defer modulesLock.Unlock() + + if _, ok := endpoints[name]; ok { + panic("Register: module with specified name is already registered: " + name) + } + + endpoints[name] = factory +} diff --git a/framework/module/storage.go b/framework/module/storage.go new file mode 100644 index 0000000..c332569 --- /dev/null +++ b/framework/module/storage.go @@ -0,0 +1,50 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + imapbackend "github.com/emersion/go-imap/backend" +) + +// Storage interface is a slightly modified go-imap's Backend interface +// (authentication is removed). +// +// Modules implementing this interface should be registered with prefix +// "storage." in name. +type Storage interface { + // GetOrCreateIMAPAcct returns User associated with storage account specified by + // the name. + // + // If it doesn't exists - it should be created. + GetOrCreateIMAPAcct(username string) (imapbackend.User, error) + GetIMAPAcct(username string) (imapbackend.User, error) + + // Extensions returns list of IMAP extensions supported by backend. + IMAPExtensions() []string +} + +// ManageableStorage is an extended Storage interface that allows to +// list existing accounts, create and delete them. +type ManageableStorage interface { + Storage + + ListIMAPAccts() ([]string, error) + CreateIMAPAcct(username string) error + DeleteIMAPAcct(username string) error +} diff --git a/framework/module/table.go b/framework/module/table.go new file mode 100644 index 0000000..3e8e19e --- /dev/null +++ b/framework/module/table.go @@ -0,0 +1,43 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import "context" + +// Table is the interface implemented by module that implementation string-to-string +// translation. +// +// Modules implementing this interface should be registered with prefix +// "table." in name. +type Table interface { + Lookup(ctx context.Context, s string) (string, bool, error) +} + +// MultiTable is the interface that module can implement in addition to Table +// if it can provide multiple values as a lookup result. +type MultiTable interface { + LookupMulti(ctx context.Context, s string) ([]string, error) +} + +type MutableTable interface { + Table + Keys() ([]string, error) + RemoveKey(k string) error + SetKey(k, v string) error +} diff --git a/framework/module/tls_loader.go b/framework/module/tls_loader.go new file mode 100644 index 0000000..06184c0 --- /dev/null +++ b/framework/module/tls_loader.go @@ -0,0 +1,41 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "crypto/tls" +) + +// TLSLoader interface is module interface that can be used to supply TLS +// certificates to TLS-enabled endpoints. +// +// The interface is intentionally kept simple, all configuration and parameters +// necessary are to be provided using conventional module configuration. +// +// If loader returns multiple certificate chains - endpoint will serve them +// based on SNI matching. +// +// Note that loading function will be called for each connections - it is +// highly recommended to cache parsed form. +// +// Modules implementing this interface should be registered with prefix +// "tls.loader." in name. +type TLSLoader interface { + ConfigureTLS(c *tls.Config) error +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..668bf27 --- /dev/null +++ b/go.mod @@ -0,0 +1,173 @@ +module github.com/foxcpp/maddy + +go 1.23.1 + +toolchain go1.23.5 + +require ( + blitiri.com.ar/go/spf v1.5.1 + github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 + github.com/c0va23/go-proxyprotocol v0.9.1 + github.com/caddyserver/certmagic v0.21.7 + github.com/emersion/go-imap v1.2.2-0.20220928192137-6fac715be9cf + github.com/emersion/go-imap-compress v0.0.0-20201103190257-14809af1d1b9 + github.com/emersion/go-imap-sortthread v1.2.0 + github.com/emersion/go-message v0.18.2 + github.com/emersion/go-milter v0.4.1 + github.com/emersion/go-msgauth v0.6.8 + github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 + github.com/emersion/go-smtp v0.21.3 + github.com/foxcpp/go-dovecot-sasl v0.0.0-20200522223722-c4699d7a24bf + github.com/foxcpp/go-imap-backend-tests v0.0.0-20220105184719-e80aa29a5e16 + github.com/foxcpp/go-imap-i18nlevel v0.0.0-20200208001533-d6ec88553005 + github.com/foxcpp/go-imap-mess v0.0.0-20230108134257-b7ec3a649613 + github.com/foxcpp/go-imap-namespace v0.0.0-20200802091432-08496dd8e0ed + github.com/foxcpp/go-imap-sql v0.5.1-0.20250124140007-8da5567429d5 + github.com/foxcpp/go-mockdns v1.1.0 + github.com/foxcpp/go-mtasts v0.0.0-20240130093538-1438da2e5932 + github.com/go-ldap/ldap/v3 v3.4.10 + github.com/go-sql-driver/mysql v1.8.1 + github.com/google/uuid v1.6.0 + github.com/hashicorp/go-hclog v1.6.3 + github.com/johannesboyne/gofakes3 v0.0.0-20210704111953-6a9f95c2941c + github.com/lib/pq v1.10.9 + github.com/libdns/acmedns v0.2.0 + github.com/libdns/alidns v1.0.3 + github.com/libdns/cloudflare v0.1.1 + github.com/libdns/digitalocean v0.0.0-20230728223659-4f9064657aea + github.com/libdns/gandi v1.0.3 + github.com/libdns/gcore v0.0.0-20250127070537-4a9d185c9d20 + github.com/libdns/googleclouddns v1.1.0 + github.com/libdns/hetzner v0.0.1 + github.com/libdns/leaseweb v0.4.0 + github.com/libdns/libdns v0.2.2 + github.com/libdns/metaname v0.3.0 + github.com/libdns/namecheap v0.0.0-20211109042440-fc7440785c8e + github.com/libdns/namedotcom v0.3.3 + github.com/libdns/rfc2136 v0.1.1 + github.com/libdns/route53 v1.5.1 + github.com/libdns/vultr v1.0.0 + github.com/mattn/go-sqlite3 v1.14.24 + github.com/miekg/dns v1.1.63 + github.com/minio/minio-go/v7 v7.0.84 + github.com/netauth/netauth v0.6.2 + github.com/prometheus/client_golang v1.20.5 + github.com/urfave/cli/v2 v2.27.5 + go.uber.org/zap v1.27.0 + golang.org/x/crypto v0.32.0 + golang.org/x/net v0.34.0 + golang.org/x/sync v0.10.0 + golang.org/x/text v0.21.0 + modernc.org/sqlite v1.34.5 +) + +require ( + cloud.google.com/go/auth v0.14.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect + cloud.google.com/go/compute/metadata v0.6.0 // indirect + filippo.io/edwards25519 v1.1.0 // indirect + github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect + github.com/G-Core/gcore-dns-sdk-go v0.2.9 // indirect + github.com/aws/aws-sdk-go v1.44.40 // indirect + github.com/aws/aws-sdk-go-v2 v1.33.0 // indirect + github.com/aws/aws-sdk-go-v2/config v1.29.1 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.54 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 // indirect + github.com/aws/aws-sdk-go-v2/service/route53 v1.48.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 // indirect + github.com/aws/smithy-go v1.22.2 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/caddyserver/zerossl v0.1.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect + github.com/digitalocean/godo v1.134.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.4 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/googleapis/gax-go/v2 v2.14.1 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/magiconair/properties v1.8.9 // indirect + github.com/mailru/easyjson v0.9.0 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mholt/acmez/v3 v3.0.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/netauth/protocol v0.0.0-20210918062754-7fee492ffcbd // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/pierrec/lz4 v2.6.1+incompatible // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.12.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.19.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/vultr/govultr/v3 v3.14.1 // indirect + github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + github.com/zeebo/blake3 v0.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect + go.opentelemetry.io/otel v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.34.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap/exp v0.3.0 // indirect + golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/oauth2 v0.25.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/time v0.9.0 // indirect + golang.org/x/tools v0.29.0 // indirect + google.golang.org/api v0.218.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 // indirect + google.golang.org/grpc v1.70.0 // indirect + google.golang.org/protobuf v1.36.4 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + gotest.tools v2.2.0+incompatible // indirect + modernc.org/libc v1.61.9 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.8.2 // indirect +) + +replace github.com/emersion/go-imap => github.com/foxcpp/go-imap v1.0.0-beta.1.0.20220623182312-df940c324887 + +replace github.com/emersion/go-smtp => github.com/foxcpp/go-smtp v1.21.4-0.20250124171104-c8519ae4fb23 // v1.21.3+maddy.1 + +replace github.com/libdns/gandi => github.com/foxcpp/libdns-gandi v1.0.4-0.20240127130558-4782f9d5ce3e // v1.0.3+maddy.1 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..4d3eaea --- /dev/null +++ b/go.sum @@ -0,0 +1,1329 @@ +blitiri.com.ar/go/spf v1.5.1 h1:CWUEasc44OrANJD8CzceRnRn1Jv0LttY68cYym2/pbE= +blitiri.com.ar/go/spf v1.5.1/go.mod h1:E71N92TfL4+Yyd5lpKuE9CAF2pd4JrUq1xQfkTxoNdk= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= +cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= +cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= +cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= +cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= +cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= +cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= +cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= +cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= +cloud.google.com/go/auth v0.14.0 h1:A5C4dKV/Spdvxcl0ggWwWEzzP7AZMJSEIgrkngwhGYM= +cloud.google.com/go/auth v0.14.0/go.mod h1:CYsoRL1PdiDuqeQpZE0bP2pnPrGqFcOkI0nldEQis+A= +cloud.google.com/go/auth/oauth2adapt v0.2.7 h1:/Lc7xODdqcEw8IrZ9SvwnlLX6j9FHQM74z6cBk9Rw6M= +cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc= +cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= +cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= +cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= +cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= +cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= +cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= +cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= +cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= +cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= +cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= +cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= +cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= +cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= +cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= +cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= +cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= +cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= +cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= +cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= +cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= +cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= +cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= +cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= +cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= +cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= +cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= +cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= +cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= +cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= +cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= +cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= +cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= +cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= +cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= +cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= +cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= +cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= +cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= +cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= +cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= +cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= +cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= +cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= +cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= +cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= +cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= +cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= +cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= +cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= +cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= +cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= +cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= +cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/G-Core/gcore-dns-sdk-go v0.2.9 h1:LMMZIRX8y3aJJuAviNSpFmLbovZUw+6Om+8VElp1F90= +github.com/G-Core/gcore-dns-sdk-go v0.2.9/go.mod h1:35t795gOfzfVanhzkFyUXEzaBuMXwETmJldPpP28MN4= +github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI= +github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI= +github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.44.40 h1:MR0qefjBJrZuXE0VoeKMQFtjS2tUeVpbQNfb7NzQNgI= +github.com/aws/aws-sdk-go v1.44.40/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go-v2 v1.33.0 h1:Evgm4DI9imD81V0WwD+TN4DCwjUMdc94TrduMLbgZJs= +github.com/aws/aws-sdk-go-v2 v1.33.0/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/config v1.29.1 h1:JZhGawAyZ/EuJeBtbQYnaoftczcb2drR2Iq36Wgz4sQ= +github.com/aws/aws-sdk-go-v2/config v1.29.1/go.mod h1:7bR2YD5euaxBhzt2y/oDkt3uNRb6tjFp98GlTFueRwk= +github.com/aws/aws-sdk-go-v2/credentials v1.17.54 h1:4UmqeOqJPvdvASZWrKlhzpRahAulBfyTJQUaYy4+hEI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.54/go.mod h1:RTdfo0P0hbbTxIhmQrOsC/PquBZGabEPnCaxxKRPSnI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 h1:5grmdTdMsovn9kPZPI23Hhvp0ZyNm5cRO+IZFIYiAfw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24/go.mod h1:zqi7TVKTswH3Ozq28PkmBmgzG1tona7mo9G2IJg4Cis= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 h1:igORFSiH3bfq4lxKFkTSYDhJEUCYo6C8VKiWJjYwQuQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28/go.mod h1:3So8EA/aAYm36L7XIvCVwLa0s5N0P7o2b1oqnx/2R4g= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 h1:1mOW9zAUMhTSrMDssEHS/ajx8JcAj/IcftzcmNlmVLI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28/go.mod h1:kGlXVIWDfvt2Ox5zEaNglmq0hXPHgQFNMix33Tw22jA= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 h1:TQmKDyETFGiXVhZfQ/I0cCFziqqX58pi4tKJGYGFSz0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9/go.mod h1:HVLPK2iHQBUx7HfZeOQSEu3v2ubZaAY2YPbAm5/WUyY= +github.com/aws/aws-sdk-go-v2/service/route53 v1.48.2 h1:Rxg1R0CHxVb9ggQLufOkr4an3yFEkTDN+N5+LFU4aEg= +github.com/aws/aws-sdk-go-v2/service/route53 v1.48.2/go.mod h1:TN4PcCL0lvqmYcv+AV8iZFC4Sd0FM06QDaoBXrFEftU= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 h1:kuIyu4fTT38Kj7YCC7ouNbVZSSpqkZ+LzIfhCr6Dg+I= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.11/go.mod h1:Ro744S4fKiCCuZECXgOi760TiYylUM8ZBf6OGiZzJtY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 h1:l+dgv/64iVlQ3WsBbnn+JSbkj01jIi+SM0wYsj3y/hY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10/go.mod h1:Fzsj6lZEb8AkTE5S68OhcbBqeWPsR8RnGuKPr8Todl8= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 h1:BRVDbewN6VZcwr+FBOszDKvYeXY1kJ+GGMCcpghlw0U= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.9/go.mod h1:f6vjfZER1M17Fokn0IzssOTMT2N8ZSq+7jnNF0tArvw= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/c0va23/go-proxyprotocol v0.9.1 h1:5BCkp0fDJOhzzH1lhjUgHhmZz9VvRMMif1U2D31hb34= +github.com/c0va23/go-proxyprotocol v0.9.1/go.mod h1:TNjUV+llvk8TvWJxlPYAeAYZgSzT/iicNr3nWBWX320= +github.com/caddyserver/certmagic v0.21.7 h1:66KJioPFJwttL43KYSWk7ErSmE6LfaJgCQuhm8Sg6fg= +github.com/caddyserver/certmagic v0.21.7/go.mod h1:LCPG3WLxcnjVKl/xpjzM0gqh0knrKKKiO5WVttX2eEI= +github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA= +github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/digitalocean/godo v1.41.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= +github.com/digitalocean/godo v1.134.0 h1:dT7aQR9jxNOQEZwzP+tAYcxlj5szFZScC33+PAYGQVM= +github.com/digitalocean/godo v1.134.0/go.mod h1:PU8JB6I1XYkQIdHFop8lLAY9ojp6M0XcU0TWaQSxbrc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emersion/go-imap-appendlimit v0.0.0-20190308131241-25671c986a6a/go.mod h1:ikgISoP7pRAolqsVP64yMteJa2FIpS6ju88eBT6K1yQ= +github.com/emersion/go-imap-compress v0.0.0-20201103190257-14809af1d1b9 h1:7dmV11mle4UAQ7lX+Hdzx6akKFg3hVm/UUmQ7t6VgTQ= +github.com/emersion/go-imap-compress v0.0.0-20201103190257-14809af1d1b9/go.mod h1:2Ro1PbmiqYiRe5Ct2sGR5hHaKSVHeRpVZwXx8vyYt98= +github.com/emersion/go-imap-move v0.0.0-20180601155324-5eb20cb834bf/go.mod h1:QuMaZcKFDVI0yCrnAbPLfbwllz1wtOrZH8/vZ5yzp4w= +github.com/emersion/go-imap-sortthread v1.2.0 h1:EMVEJXPWAhXMWECjR82Rn/tza6MddcvTwGAdTu1vJKU= +github.com/emersion/go-imap-sortthread v1.2.0/go.mod h1:UhenCBupR+vSYRnqJkpjSq84INUCsyAK1MLpogv14pE= +github.com/emersion/go-message v0.15.0/go.mod h1:wQUEfE+38+7EW8p8aZ96ptg6bAb1iwdgej19uXASlE4= +github.com/emersion/go-message v0.18.0/go.mod h1:Zi69ACvzaoV/MBnrxfVBPV3xWEuCmC2nEN39oJF4B8A= +github.com/emersion/go-message v0.18.1/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= +github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg= +github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= +github.com/emersion/go-milter v0.4.1 h1:gLs9QD0zEHF8omgEw8M+aGz6iwBNpWLAcwgSur0ra4M= +github.com/emersion/go-milter v0.4.1/go.mod h1:erCQVl0mH4SX9jEvwe+wyndit0rQtmvMLH86V6NGtkI= +github.com/emersion/go-msgauth v0.6.8 h1:kW/0E9E8Zx5CdKsERC/WnAvnXvX7q9wTHia1OA4944A= +github.com/emersion/go-msgauth v0.6.8/go.mod h1:YDwuyTCUHu9xxmAeVj0eW4INnwB6NNZoPdLerpSxRrc= +github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k= +github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3DfFi4EXdT+B+gTtzx8GC9xsc26Znk= +github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/foxcpp/go-dovecot-sasl v0.0.0-20200522223722-c4699d7a24bf h1:rmBPY5fryjp9zLQYsUmQqqgsYq7qeVfrjtr96Tf9vD8= +github.com/foxcpp/go-dovecot-sasl v0.0.0-20200522223722-c4699d7a24bf/go.mod h1:5yZUmwr851vgjyAfN7OEfnrmKOh/qLA5dbGelXYsu1E= +github.com/foxcpp/go-imap v1.0.0-beta.1.0.20220623182312-df940c324887 h1:qUoaaHyrRpQw85ru6VQcC6JowdhrWl7lSbI1zRX1FTM= +github.com/foxcpp/go-imap v1.0.0-beta.1.0.20220623182312-df940c324887/go.mod h1:Qlx1FSx2FTxjnjWpIlVNEuX+ylerZQNFE5NsmKFSejY= +github.com/foxcpp/go-imap-backend-tests v0.0.0-20220105184719-e80aa29a5e16 h1:qheFPDpteiUy7Ym18R68OYenpk85UyKYGkhYTmddSBg= +github.com/foxcpp/go-imap-backend-tests v0.0.0-20220105184719-e80aa29a5e16/go.mod h1:OPP1AgKxMPo3aHX5pcEZLQhhh5sllFcB8aUN9f6a6X8= +github.com/foxcpp/go-imap-i18nlevel v0.0.0-20200208001533-d6ec88553005 h1:pfoFtkTTQ473qStSN79jhCFBWqMQt/3DQ3NGuXvT+50= +github.com/foxcpp/go-imap-i18nlevel v0.0.0-20200208001533-d6ec88553005/go.mod h1:34FwxnjC2N+EFs2wMtsHevrZLWRKRuVU8wEcHWKq/nE= +github.com/foxcpp/go-imap-mess v0.0.0-20230108134257-b7ec3a649613 h1:fw9OWfPxP1CK4D+XAEEg0JzhvFGo04L+F5Xw55t9s3E= +github.com/foxcpp/go-imap-mess v0.0.0-20230108134257-b7ec3a649613/go.mod h1:P/O/qz4gaVkefzJ40BUtN/ZzBnaEg0YYe1no/SMp7Aw= +github.com/foxcpp/go-imap-namespace v0.0.0-20200802091432-08496dd8e0ed h1:1Jo7geyvunrPSjL6F6D9EcXoNApS5v3LQaro7aUNPnE= +github.com/foxcpp/go-imap-namespace v0.0.0-20200802091432-08496dd8e0ed/go.mod h1:Shows1vmkBWO40ChOClaUe6DUnZrsP1UPAuoWzIUdgQ= +github.com/foxcpp/go-imap-sql v0.5.1-0.20250124140007-8da5567429d5 h1:jMxhw9qmwqg70qfMDWq0ImRHAduQjkTZOC9vBs5t2ug= +github.com/foxcpp/go-imap-sql v0.5.1-0.20250124140007-8da5567429d5/go.mod h1:LMlfyNkVs7v2zE6OVeGe9qWPmKFdXDmLNddPLodPVIw= +github.com/foxcpp/go-mockdns v0.0.0-20191216195825-5eabd8dbfe1f/go.mod h1:tPg4cp4nseejPd+UKxtCVQ2hUxNTZ7qQZJa7CLriIeo= +github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI= +github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= +github.com/foxcpp/go-mtasts v0.0.0-20240130093538-1438da2e5932 h1:p04U/s8IZEc+PVWIDWGUgdqGq3xsixI7XRZ6Bp/xZbQ= +github.com/foxcpp/go-mtasts v0.0.0-20240130093538-1438da2e5932/go.mod h1:RtHIZCsScdjIzXpTTjmEljtUrIjQbPBTvw7F1tKQbKk= +github.com/foxcpp/go-smtp v1.21.4-0.20250124171104-c8519ae4fb23 h1:JSnsCrRrHNBlgfKVFBxFzp3fN/wS21t8fAHcZ9B1uWI= +github.com/foxcpp/go-smtp v1.21.4-0.20250124171104-c8519ae4fb23/go.mod h1:qm27SGYgoIPRot6ubfQ/GpiPy/g3PaZAVRxiO/sDUgQ= +github.com/foxcpp/libdns-gandi v1.0.4-0.20240127130558-4782f9d5ce3e h1:hKk+CGUtwnKDGKINPEojeo91kx0tnV6V4tlzHehJPfg= +github.com/foxcpp/libdns-gandi v1.0.4-0.20240127130558-4782f9d5ce3e/go.mod h1:G6dw58Xnji2xX+lb+uZxGbtmfxKllm1CGHE2bOPG3WA= +github.com/frankban/quicktest v1.5.0/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk= +github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-ldap/ldap/v3 v3.4.10 h1:ot/iwPOhfpNVgB1o+AVXljizWZ9JTp7YF5oeyONmcJU= +github.com/go-ldap/ldap/v3 v3.4.10/go.mod h1:JXh4Uxgi40P6E9rdsYqpUtbW46D9UTjJ9QSwGRznplY= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= +github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= +github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= +github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= +github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= +github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= +github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/johannesboyne/gofakes3 v0.0.0-20210704111953-6a9f95c2941c h1:lx/uPI+mUWlqEQ9e6CtNvaK/zD64s/mQ9+yMh16PgY0= +github.com/johannesboyne/gofakes3 v0.0.0-20210704111953-6a9f95c2941c/go.mod h1:LIAXxPvcUXwOcTIj9LSNSUpE9/eMHalTWxsP/kmWxQI= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libdns/acmedns v0.2.0 h1:zTXdHZwe3r2issdVRyqt5/4X2yHpiBVmFnTrwBA29ik= +github.com/libdns/acmedns v0.2.0/go.mod h1:XlKHilQQK/IGHYY//vCb903PdG4Wc/XnDQzcMp2hV3g= +github.com/libdns/alidns v1.0.3 h1:LFHuGnbseq5+HCeGa1aW8awyX/4M2psB9962fdD2+yQ= +github.com/libdns/alidns v1.0.3/go.mod h1:e18uAG6GanfRhcJj6/tps2rCMzQJaYVcGKT+ELjdjGE= +github.com/libdns/cloudflare v0.1.1 h1:FVPfWwP8zZCqj268LZjmkDleXlHPlFU9KC4OJ3yn054= +github.com/libdns/cloudflare v0.1.1/go.mod h1:9VK91idpOjg6v7/WbjkEW49bSCxj00ALesIFDhJ8PBU= +github.com/libdns/digitalocean v0.0.0-20230728223659-4f9064657aea h1:IGlMNZCUp8Ho7NYYorpP5ZJgg2mFXARs6eHs/pSqFkA= +github.com/libdns/digitalocean v0.0.0-20230728223659-4f9064657aea/go.mod h1:B2TChhOTxvBflpRTHlguXWtwa1Ha5WI6JkB6aCViM+0= +github.com/libdns/gcore v0.0.0-20250127070537-4a9d185c9d20 h1:bQwFw+C9sX/zYZlV53ey0KnNkxrfWYIFpvptuAVhJ1Y= +github.com/libdns/gcore v0.0.0-20250127070537-4a9d185c9d20/go.mod h1:JGoT1mbmqQwtYQqN5F/vGc9j4TTTMKw/hDm5vXADHUI= +github.com/libdns/googleclouddns v1.1.0 h1:murPR1LfTZZObLV2OLxUVmymWH25glkMFKpDjkk2m0E= +github.com/libdns/googleclouddns v1.1.0/go.mod h1:3tzd056dfqKlf71V8Oy19En4WjJ3ybyuWx6P9bQSCIw= +github.com/libdns/hetzner v0.0.1 h1:WsmcsOKnfpKmzwhfyqhGQEIlEeEaEUvb7ezoJgBKaqU= +github.com/libdns/hetzner v0.0.1/go.mod h1:Jj12aJipO9Ir7OGaXueJ5J1RnerFMD0auGa6k9kujG4= +github.com/libdns/leaseweb v0.4.0 h1:WG9R5AwewpYM4goymFwnG2SB0qwL8gMsSzwRHZHee/U= +github.com/libdns/leaseweb v0.4.0/go.mod h1:dvTvEn11JN6+ebhAQ60l+jiaBiEqyJFs3EIo0YBcQkU= +github.com/libdns/libdns v0.1.0/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= +github.com/libdns/libdns v0.2.0/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= +github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= +github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s= +github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= +github.com/libdns/metaname v0.3.0 h1:HJudLYthdv52TupOPczojip/nEQHW7xqk5+whGReva4= +github.com/libdns/metaname v0.3.0/go.mod h1:a3hqEgj59tjWaWlF4WxQGhvMVtjz1E4Ngs1GfVS+VhQ= +github.com/libdns/namecheap v0.0.0-20211109042440-fc7440785c8e h1:WCcKyxiiK/sJnST1ulVBKNg4J8luCYDdgUrp2ySMO2s= +github.com/libdns/namecheap v0.0.0-20211109042440-fc7440785c8e/go.mod h1:dED6sMLZxIcilF1GjrcpwgVoCglXGMn86irqQzRhqRY= +github.com/libdns/namedotcom v0.3.3 h1:R10C7+IqQGVeC4opHHMiFNBxdNBg1bi65ZwqLESl+jE= +github.com/libdns/namedotcom v0.3.3/go.mod h1:GbYzsAF2yRUpI0WgIK5fs5UX+kDVUPaYCFLpTnKQm0s= +github.com/libdns/rfc2136 v0.1.1 h1:GKh2r08xt4aYeGlXR9eFrJMfFKD5i9QHBOpT1FIww/U= +github.com/libdns/rfc2136 v0.1.1/go.mod h1:tgXWavE+5OiAfdKxBnuG8OBEwQFAu7uuiS3+laspAGs= +github.com/libdns/route53 v1.5.1 h1:dkdcc2CKY/EHBBzAKqE0Cko7MKR8uVJ3GvpzwKu/UKM= +github.com/libdns/route53 v1.5.1/go.mod h1:joT4hKmaTNKHEwb7GmZ65eoDz1whTu7KKYPS8ZqIh6Q= +github.com/libdns/vultr v1.0.0 h1:W8B4+k2bm9ro3bZLSZV9hMOQI+uO6Svu+GmD+Olz7ZI= +github.com/libdns/vultr v1.0.0/go.mod h1:8K1HJExcbeHS4YPkFHRZpqpXZzZ+DZAA0m0VikJgEqk= +github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM= +github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/martinlindhe/base36 v1.0.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mholt/acmez/v3 v3.0.1 h1:4PcjKjaySlgXK857aTfDuRbmnM5gb3Ruz3tvoSJAUp8= +github.com/mholt/acmez/v3 v3.0.1/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ= +github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= +github.com/miekg/dns v1.1.63 h1:8M5aAw6OMZfFXTT7K5V0Eu5YiiL8l7nUAkyN6C9YwaY= +github.com/miekg/dns v1.1.63/go.mod h1:6NGHfjhpmr5lt3XPLuyfDJi5AXbNIPM9PY6H6sF1Nfs= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.84 h1:D1HVmAF8JF8Bpi6IU4V9vIEj+8pc+xU88EWMs2yed0E= +github.com/minio/minio-go/v7 v7.0.84/go.mod h1:57YXpvc5l3rjPdhqNrDsvVlY0qPI6UTk1bflAe+9doY= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/netauth/netauth v0.6.2 h1:Gtx/Xxa6YUaGny+iVvWyp+FAmtLQ1IlbB2uWTZEpWxQ= +github.com/netauth/netauth v0.6.2/go.mod h1:4PEbISVqRCQaXaDAt289w3nK9UhoF8/ZOLy31Hbv7ds= +github.com/netauth/protocol v0.0.0-20210918062754-7fee492ffcbd h1:4yVpQ/+li28lQ/daYCWeDB08obRmjaoAw2qfFFaCQ40= +github.com/netauth/protocol v0.0.0-20210918062754-7fee492ffcbd/go.mod h1:wpK5wqysOJU1w2OxgG65du8M7UqBkxzsNaJdjwiRqAs= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= +github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= +github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 h1:J6qvD6rbmOil46orKqJaRPG+zTpoGlBTUdyv8ki63L0= +github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+VKSARF5y/tS9XFSP7vWDfS+GUC5vs/YT7M5XDTUEM= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= +github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= +github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= +github.com/vultr/govultr/v3 v3.14.1 h1:9BpyZgsWasuNoR39YVMcq44MSaF576Z4D+U3ro58eJQ= +github.com/vultr/govultr/v3 v3.14.1/go.mod h1:q34Wd76upKmf+vxFMgaNMH3A8BbsPBmSYZUGC8oZa5w= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI= +github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= +go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= +go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= +go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= +go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190310074541-c10a0554eabf/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= +golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= +golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= +google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.218.0 h1:x6JCjEWeZ9PFCRe9z0FBrNwj7pB7DOAqT35N+IPnAUA= +google.golang.org/api v0.218.0/go.mod h1:5VGHBAkxrA/8EFjLVEYmMUJ8/8+gWWQ3s4cFH0FxG2M= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= +google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= +google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= +google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221018160656-63c7b68cfc55/go.mod h1:45EK0dUbEZ2NHjCeAd2LXmyjAgGUGrpGROgjhC3ADck= +google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 h1:91mG8dNTpkC0uChJUQ9zCiRqx3GEEFOWaRZ0mI6Oj2I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= +google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= +google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0= +modernc.org/cc/v4 v4.24.4/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.23.13 h1:PFiaemQwE/jdwi8XEHyEV+qYWoIuikLP3T4rvDeJb00= +modernc.org/ccgo/v4 v4.23.13/go.mod h1:vdN4h2WR5aEoNondUx26K7G8X+nuBscYnAEWSRmN2/0= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.6.1 h1:+Qf6xdG8l7B27TQ8D8lw/iFMUj1RXRBOuMUWziJOsk8= +modernc.org/gc/v2 v2.6.1/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/libc v1.61.9 h1:PLSBXVkifXGELtJ5BOnBUyAHr7lsatNwFU/RRo4kfJM= +modernc.org/libc v1.61.9/go.mod h1:61xrnzk/aR8gr5bR7Uj/lLFLuXu2/zMpIjcry63Eumk= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.8.2 h1:cL9L4bcoAObu4NkxOlKWBWtNHIsnnACGF/TbqQ6sbcI= +modernc.org/memory v1.8.2/go.mod h1:ZbjSvMO5NQ1A2i3bWeDiVMxIorXwdClKE/0SZ+BMotU= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/README.md b/internal/README.md new file mode 100644 index 0000000..da5d57c --- /dev/null +++ b/internal/README.md @@ -0,0 +1,24 @@ +maddy source tree +------------------ + +Main maddy code base lives here. No packages are intended to be used in +third-party software hence API is not stable. + +Subdirectories are organized as follows: +``` +/ + auxiliary libraries +endpoint/ + modules - protocol listeners (e.g. SMTP server, etc) +target/ + modules - final delivery targets (including outbound delivery, such as + target.smtp, remote) +auth/ + modules - authentication providers +check/ + modules - message checkers (module.Check) +modify/ + modules - message modifiers (module.Modifier) +storage/ + modules - local messages storage implementations (module.Storage) +``` diff --git a/internal/auth/auth.go b/internal/auth/auth.go new file mode 100644 index 0000000..4df144c --- /dev/null +++ b/internal/auth/auth.go @@ -0,0 +1,53 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package auth + +import "strings" + +func CheckDomainAuth(username string, perDomain bool, allowedDomains []string) (loginName string, allowed bool) { + var accountName, domain string + if perDomain { + parts := strings.Split(username, "@") + if len(parts) != 2 { + return "", false + } + domain = parts[1] + accountName = username + } else { + parts := strings.Split(username, "@") + accountName = parts[0] + if len(parts) == 2 { + domain = parts[1] + } + } + + allowed = domain == "" + if allowedDomains != nil && domain != "" { + for _, allowedDomain := range allowedDomains { + if strings.EqualFold(domain, allowedDomain) { + allowed = true + } + } + if !allowed { + return "", false + } + } + + return accountName, allowed +} diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go new file mode 100644 index 0000000..65ffee4 --- /dev/null +++ b/internal/auth/auth_test.go @@ -0,0 +1,92 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package auth + +import ( + "fmt" + "testing" +) + +func TestCheckDomainAuth(t *testing.T) { + cases := []struct { + rawUsername string + + perDomain bool + allowedDomains []string + + loginName string + }{ + { + rawUsername: "username", + loginName: "username", + }, + { + rawUsername: "username", + allowedDomains: []string{"example.org"}, + loginName: "username", + }, + { + rawUsername: "username@example.org", + allowedDomains: []string{"example.org"}, + loginName: "username", + }, + { + rawUsername: "username@example.com", + allowedDomains: []string{"example.org"}, + }, + { + rawUsername: "username", + allowedDomains: []string{"example.org"}, + perDomain: true, + }, + { + rawUsername: "username@example.com", + allowedDomains: []string{"example.org"}, + perDomain: true, + }, + { + rawUsername: "username@EXAMPLE.Org", + allowedDomains: []string{"exaMPle.org"}, + perDomain: true, + loginName: "username@EXAMPLE.Org", + }, + { + rawUsername: "username@example.org", + allowedDomains: []string{"example.org"}, + perDomain: true, + loginName: "username@example.org", + }, + } + + for _, case_ := range cases { + t.Run(fmt.Sprintf("%+v", case_), func(t *testing.T) { + loginName, allowed := CheckDomainAuth(case_.rawUsername, case_.perDomain, case_.allowedDomains) + if case_.loginName != "" && !allowed { + t.Fatalf("Unexpected authentication fail") + } + if case_.loginName == "" && allowed { + t.Fatalf("Expected authentication fail, got %s as login name", loginName) + } + + if loginName != case_.loginName { + t.Errorf("Incorrect login name, got %s, wanted %s", loginName, case_.loginName) + } + }) + } +} diff --git a/internal/auth/dovecot_sasl/dovecot_sasl.go b/internal/auth/dovecot_sasl/dovecot_sasl.go new file mode 100644 index 0000000..c7dd6cc --- /dev/null +++ b/internal/auth/dovecot_sasl/dovecot_sasl.go @@ -0,0 +1,160 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dovecotsasl + +import ( + "fmt" + "net" + + "github.com/emersion/go-sasl" + dovecotsasl "github.com/foxcpp/go-dovecot-sasl" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth" +) + +type Auth struct { + instName string + serverEndpoint string + log log.Logger + + network string + addr string + + mechanisms map[string]dovecotsasl.Mechanism +} + +const modName = "dovecot_sasl" + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + a := &Auth{ + instName: instName, + log: log.Logger{Name: modName, Debug: log.DefaultLogger.Debug}, + } + + switch len(inlineArgs) { + case 0: + case 1: + a.serverEndpoint = inlineArgs[0] + default: + return nil, fmt.Errorf("%s: one or none arguments needed", modName) + } + + return a, nil +} + +func (a *Auth) Name() string { + return modName +} + +func (a *Auth) InstanceName() string { + return a.instName +} + +func (a *Auth) getConn() (*dovecotsasl.Client, error) { + // TODO: Connection pooling + conn, err := net.Dial(a.network, a.addr) + if err != nil { + return nil, fmt.Errorf("%s: unable to contact server: %v", modName, err) + } + + cl, err := dovecotsasl.NewClient(conn) + if err != nil { + return nil, fmt.Errorf("%s: unable to contact server: %v", modName, err) + } + + return cl, nil +} + +func (a *Auth) returnConn(cl *dovecotsasl.Client) { + cl.Close() +} + +func (a *Auth) Init(cfg *config.Map) error { + cfg.String("endpoint", false, false, a.serverEndpoint, &a.serverEndpoint) + if _, err := cfg.Process(); err != nil { + return err + } + if a.serverEndpoint == "" { + return fmt.Errorf("%s: missing server endpoint", modName) + } + + endp, err := config.ParseEndpoint(a.serverEndpoint) + if err != nil { + return fmt.Errorf("%s: invalid server endpoint: %v", modName, err) + } + + // Dial once to check usability and also to get list of mechanisms. + conn, err := net.Dial(endp.Scheme, endp.Address()) + if err != nil { + return fmt.Errorf("%s: unable to contact server: %v", modName, err) + } + + cl, err := dovecotsasl.NewClient(conn) + if err != nil { + return fmt.Errorf("%s: unable to contact server: %v", modName, err) + } + + defer cl.Close() + a.mechanisms = make(map[string]dovecotsasl.Mechanism, len(cl.ConnInfo().Mechs)) + for name, mech := range cl.ConnInfo().Mechs { + if mech.Private { + continue + } + a.mechanisms[name] = mech + } + + a.network = endp.Scheme + a.addr = endp.Address() + + return nil +} + +func (a *Auth) AuthPlain(username, password string) error { + if _, ok := a.mechanisms[sasl.Plain]; ok { + cl, err := a.getConn() + if err != nil { + return exterrors.WithTemporary(err, true) + } + defer a.returnConn(cl) + + // Pretend it is SMTPS even though we really don't know. + // We also have no connection information to pass to the server... + return cl.Do("SMTP", sasl.NewPlainClient("", username, password), + dovecotsasl.Secured, dovecotsasl.NoPenalty) + } + if _, ok := a.mechanisms[sasl.Login]; ok { + cl, err := a.getConn() + if err != nil { + return err + } + defer a.returnConn(cl) + + return cl.Do("SMTP", sasl.NewLoginClient(username, password), + dovecotsasl.Secured, dovecotsasl.NoPenalty) + } + + return auth.ErrUnsupportedMech +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/auth/external/externalauth.go b/internal/auth/external/externalauth.go new file mode 100644 index 0000000..59d71fb --- /dev/null +++ b/internal/auth/external/externalauth.go @@ -0,0 +1,103 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package external + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth" +) + +type ExternalAuth struct { + modName string + instName string + helperPath string + + perDomain bool + domains []string + + Log log.Logger +} + +func NewExternalAuth(modName, instName string, _, inlineArgs []string) (module.Module, error) { + ea := &ExternalAuth{ + modName: modName, + instName: instName, + Log: log.Logger{Name: modName}, + } + + if len(inlineArgs) != 0 { + return nil, errors.New("external: inline arguments are not used") + } + + return ea, nil +} + +func (ea *ExternalAuth) Name() string { + return ea.modName +} + +func (ea *ExternalAuth) InstanceName() string { + return ea.instName +} + +func (ea *ExternalAuth) Init(cfg *config.Map) error { + cfg.Bool("debug", false, false, &ea.Log.Debug) + cfg.Bool("perdomain", false, false, &ea.perDomain) + cfg.StringList("domains", false, false, nil, &ea.domains) + cfg.String("helper", false, false, "", &ea.helperPath) + if _, err := cfg.Process(); err != nil { + return err + } + if ea.perDomain && ea.domains == nil { + return errors.New("auth_domains must be set if auth_perdomain is used") + } + + if ea.helperPath != "" { + ea.Log.Debugln("using helper:", ea.helperPath) + } else { + ea.helperPath = filepath.Join(config.LibexecDirectory, "maddy-auth-helper") + } + if _, err := os.Stat(ea.helperPath); err != nil { + return fmt.Errorf("%s doesn't exist", ea.helperPath) + } + + ea.Log.Debugln("using helper:", ea.helperPath) + + return nil +} + +func (ea *ExternalAuth) AuthPlain(username, password string) error { + accountName, ok := auth.CheckDomainAuth(username, ea.perDomain, ea.domains) + if !ok { + return module.ErrUnknownCredentials + } + + return AuthUsingHelper(ea.helperPath, accountName, password) +} + +func init() { + module.Register("auth.external", NewExternalAuth) +} diff --git a/internal/auth/external/helperauth.go b/internal/auth/external/helperauth.go new file mode 100644 index 0000000..901d57d --- /dev/null +++ b/internal/auth/external/helperauth.go @@ -0,0 +1,55 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package external + +import ( + "fmt" + "io" + "os/exec" + + "github.com/foxcpp/maddy/framework/module" +) + +func AuthUsingHelper(binaryPath, accountName, password string) error { + cmd := exec.Command(binaryPath) + stdin, err := cmd.StdinPipe() + if err != nil { + return fmt.Errorf("helperauth: stdin init: %w", err) + } + if err := cmd.Start(); err != nil { + return fmt.Errorf("helperauth: process start: %w", err) + } + if _, err := io.WriteString(stdin, accountName+"\n"); err != nil { + return fmt.Errorf("helperauth: stdin write: %w", err) + } + if _, err := io.WriteString(stdin, password+"\n"); err != nil { + return fmt.Errorf("helperauth: stdin write: %w", err) + } + if err := cmd.Wait(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + // Exit code 1 is for authentication failure. + if exitErr.ExitCode() != 1 { + return fmt.Errorf("helperauth: %w: %v", err, string(exitErr.Stderr)) + } + return module.ErrUnknownCredentials + } + return fmt.Errorf("helperauth: process wait: %w", err) + } + return nil +} diff --git a/internal/auth/ldap/ldap.go b/internal/auth/ldap/ldap.go new file mode 100644 index 0000000..04cfe9f --- /dev/null +++ b/internal/auth/ldap/ldap.go @@ -0,0 +1,289 @@ +package ldap + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/url" + "strings" + "sync" + "time" + + "github.com/foxcpp/maddy/framework/config" + tls2 "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/go-ldap/ldap/v3" +) + +const modName = "auth.ldap" + +type Auth struct { + instName string + + urls []string + readBind func(*ldap.Conn) error + startls bool + tlsCfg tls.Config + dialer *net.Dialer + requestTimeout time.Duration + + dnTemplate string + // or + baseDN string + filterTemplate string + + conn *ldap.Conn + connLock sync.Mutex + + log log.Logger +} + +func New(modName, instName string, _, inlineArgs []string) (module.Module, error) { + return &Auth{ + instName: instName, + log: log.Logger{Name: modName}, + urls: inlineArgs, + }, nil +} + +func (a *Auth) Init(cfg *config.Map) error { + a.dialer = &net.Dialer{} + + cfg.Bool("debug", true, false, &a.log.Debug) + cfg.Custom("tls_client", true, false, func() (interface{}, error) { + return tls.Config{}, nil + }, tls2.TLSClientBlock, &a.tlsCfg) + cfg.Callback("urls", func(m *config.Map, node config.Node) error { + a.urls = append(a.urls, node.Args...) + return nil + }) + cfg.Custom("bind", false, false, func() (interface{}, error) { + return func(*ldap.Conn) error { + return nil + }, nil + }, readBindDirective, &a.readBind) + cfg.Bool("starttls", false, false, &a.startls) + cfg.Duration("connect_timeout", false, false, time.Minute, &a.dialer.Timeout) + cfg.Duration("request_timeout", false, false, time.Minute, &a.requestTimeout) + cfg.String("dn_template", false, false, "", &a.dnTemplate) + cfg.String("base_dn", false, false, "", &a.baseDN) + cfg.String("filter", false, false, "", &a.filterTemplate) + if _, err := cfg.Process(); err != nil { + return err + } + + if a.dnTemplate == "" { + if a.baseDN == "" { + return fmt.Errorf("auth.ldap: base_dn not set") + } + if a.filterTemplate == "" { + return fmt.Errorf("auth.ldap: filter not set") + } + } else { + if a.baseDN != "" || a.filterTemplate != "" { + return fmt.Errorf("auth.ldap: search directives set when dn_template is used") + } + } + + if module.NoRun { + return nil + } + + var err error + a.conn, err = a.newConn() + if err != nil { + return fmt.Errorf("auth.ldap: %w", err) + } + return nil +} + +func readBindDirective(c *config.Map, n config.Node) (interface{}, error) { + if len(n.Args) == 0 { + return nil, fmt.Errorf("auth.ldap: auth expects at least one argument") + } + switch n.Args[0] { + case "off": + return func(*ldap.Conn) error { return nil }, nil + case "unauth": + if len(n.Args) == 2 { + return func(c *ldap.Conn) error { + return c.UnauthenticatedBind(n.Args[1]) + }, nil + } + return func(c *ldap.Conn) error { + return c.UnauthenticatedBind("") + }, nil + case "plain": + if len(n.Args) != 3 { + return nil, fmt.Errorf("auth.ldap: username and password expected for plaintext bind") + } + return func(c *ldap.Conn) error { + return c.Bind(n.Args[1], n.Args[2]) + }, nil + case "external": + return (*ldap.Conn).ExternalBind, nil + } + return nil, fmt.Errorf("auth.ldap: unknown bind authentication: %v", n.Args[0]) +} + +func (a *Auth) Name() string { + return modName +} + +func (a *Auth) InstanceName() string { + return a.instName +} + +func (a *Auth) newConn() (*ldap.Conn, error) { + var ( + conn *ldap.Conn + tlsCfg *tls.Config + ) + for _, u := range a.urls { + parsedURL, err := url.Parse(u) + if err != nil { + return nil, fmt.Errorf("auth.ldap: invalid server URL: %w", err) + } + hostname := parsedURL.Host + a.tlsCfg.ServerName = strings.Split(hostname, ":")[0] + tlsCfg = a.tlsCfg.Clone() + + conn, err = ldap.DialURL(u, ldap.DialWithDialer(a.dialer), ldap.DialWithTLSConfig(tlsCfg)) + if err != nil { + a.log.Error("cannot contact directory server", err, "url", u) + continue + } + break + } + if conn == nil { + return nil, fmt.Errorf("auth.ldap: all directory servers are unreachable") + } + + if a.requestTimeout != 0 { + conn.SetTimeout(a.requestTimeout) + } + + if a.startls { + if err := conn.StartTLS(tlsCfg); err != nil { + return nil, fmt.Errorf("auth.ldap: %w", err) + } + } + + if err := a.readBind(conn); err != nil { + return nil, fmt.Errorf("auth.ldap: %w", err) + } + + return conn, nil +} + +func (a *Auth) getConn() (*ldap.Conn, error) { + a.connLock.Lock() + if a.conn == nil { + conn, err := a.newConn() + if err != nil { + a.connLock.Unlock() + return nil, err + } + a.conn = conn + } + if a.conn.IsClosing() { + a.conn.Close() + conn, err := a.newConn() + if err != nil { + a.connLock.Unlock() + return nil, err + } + a.conn = conn + } + return a.conn, nil +} + +func (a *Auth) returnConn(conn *ldap.Conn) { + defer a.connLock.Unlock() + if err := a.readBind(conn); err != nil { + a.log.Error("failed to rebind for reading", err) + conn.Close() + a.conn = nil + } + if a.conn != conn { + a.conn.Close() + } + a.conn = conn +} + +func (a *Auth) Lookup(_ context.Context, username string) (string, bool, error) { + conn, err := a.getConn() + if err != nil { + return "", false, err + } + defer a.returnConn(conn) + + var userDN string + if a.dnTemplate != "" { + return "", false, fmt.Errorf("auth.ldap: lookups require search config but dn_template is used") + } else { + req := ldap.NewSearchRequest( + a.baseDN, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, + 2, 0, false, + strings.ReplaceAll(a.filterTemplate, "{username}", username), + []string{"dn"}, nil) + res, err := conn.Search(req) + if err != nil { + return "", false, fmt.Errorf("auth.ldap: search: %w", err) + } + if len(res.Entries) > 1 { + return "", false, fmt.Errorf("auth.ldap: too manu entries returned (%d)", len(res.Entries)) + } + if len(res.Entries) == 0 { + return "", false, nil + } + userDN = res.Entries[0].DN + } + + return userDN, true, nil +} + +func (a *Auth) AuthPlain(username, password string) error { + conn, err := a.getConn() + if err != nil { + return err + } + defer a.returnConn(conn) + + var userDN string + if a.dnTemplate != "" { + userDN = strings.ReplaceAll(a.dnTemplate, "{username}", username) + } else { + req := ldap.NewSearchRequest( + a.baseDN, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, + 2, 0, false, + strings.ReplaceAll(a.filterTemplate, "{username}", username), + []string{"dn"}, nil) + res, err := conn.Search(req) + if err != nil { + return fmt.Errorf("auth.ldap: search: %w", err) + } + if len(res.Entries) > 1 { + return fmt.Errorf("auth.ldap: too manu entries returned (%d)", len(res.Entries)) + } + if len(res.Entries) == 0 { + return module.ErrUnknownCredentials + } + userDN = res.Entries[0].DN + } + + if err := conn.Bind(userDN, password); err != nil { + return module.ErrUnknownCredentials + } + + return nil +} + +func init() { + var _ module.PlainAuth = &Auth{} + var _ module.Table = &Auth{} + module.Register(modName, New) + module.Register("table.ldap", New) +} diff --git a/internal/auth/netauth/netauth.go b/internal/auth/netauth/netauth.go new file mode 100644 index 0000000..3348eef --- /dev/null +++ b/internal/auth/netauth/netauth.go @@ -0,0 +1,117 @@ +package netauth + +import ( + "context" + "fmt" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/hashicorp/go-hclog" + "github.com/netauth/netauth/pkg/netauth" +) + +const modName = "auth.netauth" + +func init() { + var _ module.PlainAuth = &Auth{} + var _ module.Table = &Auth{} + module.Register(modName, New) + module.Register("table.netauth", New) +} + +// Auth binds all methods related to the NetAuth client library. +type Auth struct { + instName string + mustGroup string + + nacl *netauth.Client + + log log.Logger +} + +// New creates a new instance of the NetAuth module. +func New(modName, instName string, _, inlineArgs []string) (module.Module, error) { + return &Auth{ + instName: instName, + log: log.Logger{Name: modName}, + }, nil +} + +// Init performs deferred initialization actions. +func (a *Auth) Init(cfg *config.Map) error { + l := hclog.New(&hclog.LoggerOptions{Output: a.log}) + n, err := netauth.NewWithLog(l) + if err != nil { + return err + } + a.nacl = n + a.nacl.SetServiceName("maddy") + cfg.String("require_group", false, false, "", &a.mustGroup) + cfg.Bool("debug", true, false, &a.log.Debug) + if _, err := cfg.Process(); err != nil { + return err + } + + a.log.Debugln("Debug logging enabled") + a.log.Debugf("mustGroups status: %s", a.mustGroup) + return nil +} + +// Name returns "auth.netauth" as the fixed module name. +func (a *Auth) Name() string { + return modName +} + +// InstanceName returns the configured name for this instance of the +// plugin. Given the way that NetAuth works it doesn't really make +// sense to have more than one instance, but this is part of the API. +func (a *Auth) InstanceName() string { + return a.instName +} + +// Lookup requests the entity from the remote NetAuth server, +// potentially returning that the user does not exist at all. +func (a *Auth) Lookup(ctx context.Context, username string) (string, bool, error) { + e, err := a.nacl.EntityInfo(ctx, username) + if err != nil { + return "", false, fmt.Errorf("%s: search: %w", modName, err) + } + + if a.mustGroup != "" { + if err := a.checkMustGroup(username); err != nil { + return "", false, err + } + } + return e.GetID(), true, nil +} + +// AuthPlain attempts straightforward authentication of the entity on +// the remote NetAuth server. +func (a *Auth) AuthPlain(username, password string) error { + a.log.Debugf("attempting to auth user: %s", username) + if err := a.nacl.AuthEntity(context.Background(), username, password); err != nil { + return module.ErrUnknownCredentials + } + a.log.Debugln("netauth returns successful auth") + if a.mustGroup != "" { + if err := a.checkMustGroup(username); err != nil { + return err + } + } + return nil +} + +func (a *Auth) checkMustGroup(username string) error { + a.log.Debugf("Performing require_group check: must=%s", a.mustGroup) + groups, err := a.nacl.EntityGroups(context.Background(), username) + if err != nil { + return fmt.Errorf("%s: groups: %w", modName, err) + } + for _, g := range groups { + if g.GetName() == a.mustGroup { + return nil + } + } + return fmt.Errorf("%s: missing required group (%s not in %s)", modName, username, a.mustGroup) +} diff --git a/internal/auth/pam/module.go b/internal/auth/pam/module.go new file mode 100644 index 0000000..c93269d --- /dev/null +++ b/internal/auth/pam/module.go @@ -0,0 +1,94 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package pam + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth/external" +) + +type Auth struct { + instName string + useHelper bool + helperPath string + + Log log.Logger +} + +func New(modName, instName string, _, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, errors.New("pam: inline arguments are not used") + } + return &Auth{ + instName: instName, + Log: log.Logger{Name: modName}, + }, nil +} + +func (a *Auth) Name() string { + return "pam" +} + +func (a *Auth) InstanceName() string { + return a.instName +} + +func (a *Auth) Init(cfg *config.Map) error { + cfg.Bool("debug", true, false, &a.Log.Debug) + cfg.Bool("use_helper", false, false, &a.useHelper) + if _, err := cfg.Process(); err != nil { + return err + } + if !canCallDirectly && !a.useHelper { + return errors.New("pam: this build lacks support for direct libpam invocation, use helper binary") + } + + if a.useHelper { + a.helperPath = filepath.Join(config.LibexecDirectory, "maddy-pam-helper") + if _, err := os.Stat(a.helperPath); err != nil { + return fmt.Errorf("pam: no helper binary (maddy-pam-helper) found in %s", config.LibexecDirectory) + } + } + + return nil +} + +func (a *Auth) AuthPlain(username, password string) error { + if a.useHelper { + if err := external.AuthUsingHelper(a.helperPath, username, password); err != nil { + return err + } + } + err := runPAMAuth(username, password) + if err != nil { + return err + } + return nil +} + +func init() { + module.Register("auth.pam", New) +} diff --git a/internal/auth/pam/pam.c b/internal/auth/pam/pam.c new file mode 100644 index 0000000..38e9942 --- /dev/null +++ b/internal/auth/pam/pam.c @@ -0,0 +1,102 @@ +//+build libpam + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2022 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include "pam.h" + +static int conv_func(int num_msg, const struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) { + struct pam_response *reply = malloc(sizeof(struct pam_response)); + if (reply == NULL) { + return PAM_CONV_ERR; + } + + char* password_cpy = malloc(strlen((char*)appdata_ptr)+1); + if (password_cpy == NULL) { + return PAM_CONV_ERR; + } + memcpy(password_cpy, (char*)appdata_ptr, strlen((char*)appdata_ptr)+1); + + reply->resp = password_cpy; + reply->resp_retcode = 0; + + // PAM frees pam_response for us. + *resp = reply; + + return PAM_SUCCESS; +} + +struct error_obj run_pam_auth(const char *username, char *password) { + const struct pam_conv local_conv = { conv_func, password }; + pam_handle_t *local_auth = NULL; + int status = pam_start("maddy", username, &local_conv, &local_auth); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + ret_val.status = 2; + ret_val.func_name = "pam_start"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + status = pam_authenticate(local_auth, PAM_SILENT|PAM_DISALLOW_NULL_AUTHTOK); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + if (status == PAM_AUTH_ERR || status == PAM_USER_UNKNOWN) { + ret_val.status = 1; + } else { + ret_val.status = 2; + } + ret_val.func_name = "pam_authenticate"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + status = pam_acct_mgmt(local_auth, PAM_SILENT|PAM_DISALLOW_NULL_AUTHTOK); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + if (status == PAM_AUTH_ERR || status == PAM_USER_UNKNOWN || status == PAM_NEW_AUTHTOK_REQD) { + ret_val.status = 1; + } else { + ret_val.status = 2; + } + ret_val.func_name = "pam_acct_mgmt"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + status = pam_end(local_auth, status); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + ret_val.status = 2; + ret_val.func_name = "pam_end"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + struct error_obj ret_val; + ret_val.status = 0; + ret_val.func_name = NULL; + ret_val.error_msg = NULL; + return ret_val; +} + diff --git a/internal/auth/pam/pam.go b/internal/auth/pam/pam.go new file mode 100644 index 0000000..2b5b0ef --- /dev/null +++ b/internal/auth/pam/pam.go @@ -0,0 +1,56 @@ +//go:build cgo && libpam +// +build cgo,libpam + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package pam + +/* +#cgo LDFLAGS: -lpam +#cgo CFLAGS: -DCGO -Wall -Wextra -Werror -Wno-unused-parameter -Wno-error=unused-parameter -Wpedantic -std=c99 + +#include +#include "pam.h" +*/ +import "C" + +import ( + "errors" + "fmt" + "unsafe" +) + +const canCallDirectly = true + +var ErrInvalidCredentials = errors.New("pam: invalid credentials or unknown user") + +func runPAMAuth(username, password string) error { + usernameC := C.CString(username) + passwordC := C.CString(password) + defer C.free(unsafe.Pointer(usernameC)) + defer C.free(unsafe.Pointer(passwordC)) + errObj := C.run_pam_auth(usernameC, passwordC) + if errObj.status == 1 { + return ErrInvalidCredentials + } + if errObj.status == 2 { + return fmt.Errorf("%s: %s", C.GoString(errObj.func_name), C.GoString(errObj.error_msg)) + } + return nil +} diff --git a/internal/auth/pam/pam.h b/internal/auth/pam/pam.h new file mode 100644 index 0000000..e9831ec --- /dev/null +++ b/internal/auth/pam/pam.h @@ -0,0 +1,27 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +#pragma once + +struct error_obj { + int status; + const char* func_name; + const char* error_msg; +}; + +struct error_obj run_pam_auth(const char *username, char *password); diff --git a/internal/auth/pam/pam_stub.go b/internal/auth/pam/pam_stub.go new file mode 100644 index 0000000..fb75421 --- /dev/null +++ b/internal/auth/pam/pam_stub.go @@ -0,0 +1,34 @@ +//go:build !cgo || !libpam +// +build !cgo !libpam + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package pam + +import ( + "errors" +) + +const canCallDirectly = false + +var ErrInvalidCredentials = errors.New("pam: invalid credentials or unknown user") + +func runPAMAuth(username, password string) error { + return errors.New("pam: Can't call libpam directly") +} diff --git a/internal/auth/pass_table/hash.go b/internal/auth/pass_table/hash.go new file mode 100644 index 0000000..94d151c --- /dev/null +++ b/internal/auth/pass_table/hash.go @@ -0,0 +1,180 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package pass_table + +import ( + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "fmt" + "io" + "strconv" + "strings" + + "golang.org/x/crypto/argon2" + "golang.org/x/crypto/bcrypt" +) + +const ( + HashSHA256 = "sha256" + HashBcrypt = "bcrypt" + HashArgon2 = "argon2" + + DefaultHash = HashBcrypt + + Argon2Salt = 16 + Argon2Size = 64 +) + +type ( + // HashOpts is the structure that holds additional parameters for used hash + // functions. They are used for new passwords. + // + // These parameters should be stored together with the hashed password + // so it can be verified independently of the used HashOpts. + HashOpts struct { + // Bcrypt cost value to use. Should be at least 10. + BcryptCost int + + Argon2Time uint32 + Argon2Memory uint32 + Argon2Threads uint8 + } + + FuncHashCompute func(opts HashOpts, pass string) (string, error) + FuncHashVerify func(pass, hashSalt string) error +) + +var ( + HashCompute = map[string]FuncHashCompute{ + HashBcrypt: computeBcrypt, + HashArgon2: computeArgon2, + } + HashVerify = map[string]FuncHashVerify{ + HashBcrypt: verifyBcrypt, + HashArgon2: verifyArgon2, + } + + Hashes = []string{HashSHA256, HashBcrypt, HashArgon2} +) + +func computeArgon2(opts HashOpts, pass string) (string, error) { + salt := make([]byte, Argon2Salt) + if _, err := io.ReadFull(rand.Reader, salt); err != nil { + return "", fmt.Errorf("pass_table: failed to generate salt: %w", err) + } + + hash := argon2.IDKey([]byte(pass), salt, opts.Argon2Time, opts.Argon2Memory, opts.Argon2Threads, Argon2Size) + var out strings.Builder + out.WriteString(strconv.FormatUint(uint64(opts.Argon2Time), 10)) + out.WriteRune(':') + out.WriteString(strconv.FormatUint(uint64(opts.Argon2Memory), 10)) + out.WriteRune(':') + out.WriteString(strconv.FormatUint(uint64(opts.Argon2Threads), 10)) + out.WriteRune(':') + out.WriteString(base64.StdEncoding.EncodeToString(salt)) + out.WriteRune(':') + out.WriteString(base64.StdEncoding.EncodeToString(hash)) + return out.String(), nil +} + +func verifyArgon2(pass, hashSalt string) error { + parts := strings.SplitN(hashSalt, ":", 5) + + time, err := strconv.ParseUint(parts[0], 10, 32) + if err != nil { + return fmt.Errorf("pass_table: malformed hash string: %w", err) + } + memory, err := strconv.ParseUint(parts[1], 10, 32) + if err != nil { + return fmt.Errorf("pass_table: malformed hash string: %w", err) + } + threads, err := strconv.ParseUint(parts[2], 10, 8) + if err != nil { + return fmt.Errorf("pass_table: malformed hash string: %w", err) + } + salt, err := base64.StdEncoding.DecodeString(parts[3]) + if err != nil { + return fmt.Errorf("pass_table: malformed hash string: %w", err) + } + hash, err := base64.StdEncoding.DecodeString(parts[4]) + if err != nil { + return fmt.Errorf("pass_table: malformed hash string: %w", err) + } + + passHash := argon2.IDKey([]byte(pass), salt, uint32(time), uint32(memory), uint8(threads), Argon2Size) + if subtle.ConstantTimeCompare(passHash, hash) != 1 { + return fmt.Errorf("pass_table: hash mismatch") + } + return nil +} + +func computeSHA256(_ HashOpts, pass string) (string, error) { + salt := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, salt); err != nil { + return "", fmt.Errorf("pass_table: failed to generate salt: %w", err) + } + + hashInput := salt + hashInput = append(hashInput, []byte(pass)...) + sum := sha256.Sum256(hashInput) + return base64.StdEncoding.EncodeToString(salt) + ":" + base64.StdEncoding.EncodeToString(sum[:]), nil +} + +func verifySHA256(pass, hashSalt string) error { + parts := strings.Split(hashSalt, ":") + if len(parts) != 2 { + return fmt.Errorf("pass_table: malformed hash string, no salt") + } + salt, err := base64.StdEncoding.DecodeString(parts[0]) + if err != nil { + return fmt.Errorf("pass_table: malformed hash string, cannot decode pass: %w", err) + } + hash, err := base64.StdEncoding.DecodeString(parts[1]) + if err != nil { + return fmt.Errorf("pass_table: malformed hash string, cannot decode pass: %w", err) + } + + hashInput := salt + hashInput = append(hashInput, []byte(pass)...) + sum := sha256.Sum256(hashInput) + + if subtle.ConstantTimeCompare(sum[:], hash) != 1 { + return fmt.Errorf("pass_table: hash mismatch") + } + return nil +} + +func computeBcrypt(opts HashOpts, pass string) (string, error) { + hash, err := bcrypt.GenerateFromPassword([]byte(pass), opts.BcryptCost) + if err != nil { + return "", err + } + return string(hash), nil +} + +func verifyBcrypt(pass, hashSalt string) error { + return bcrypt.CompareHashAndPassword([]byte(hashSalt), []byte(pass)) +} + +func addSHA256() { + HashCompute[HashSHA256] = computeSHA256 + HashVerify[HashSHA256] = verifySHA256 +} diff --git a/internal/auth/pass_table/table.go b/internal/auth/pass_table/table.go new file mode 100644 index 0000000..4d0e313 --- /dev/null +++ b/internal/auth/pass_table/table.go @@ -0,0 +1,198 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package pass_table + +import ( + "context" + "fmt" + "strings" + + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/module" + "golang.org/x/crypto/bcrypt" + "golang.org/x/text/secure/precis" +) + +type Auth struct { + modName string + instName string + inlineArgs []string + + table module.Table +} + +func New(modName, instName string, _, inlineArgs []string) (module.Module, error) { + return &Auth{ + modName: modName, + instName: instName, + inlineArgs: inlineArgs, + }, nil +} + +func (a *Auth) Init(cfg *config.Map) error { + if len(a.inlineArgs) != 0 { + return modconfig.ModuleFromNode("table", a.inlineArgs, cfg.Block, cfg.Globals, &a.table) + } + + cfg.Custom("table", false, true, nil, modconfig.TableDirective, &a.table) + _, err := cfg.Process() + return err +} + +func (a *Auth) Name() string { + return a.modName +} + +func (a *Auth) InstanceName() string { + return a.instName +} + +func (a *Auth) Lookup(ctx context.Context, username string) (string, bool, error) { + key, err := precis.UsernameCaseMapped.CompareKey(username) + if err != nil { + return "", false, err + } + + return a.table.Lookup(ctx, key) +} + +func (a *Auth) AuthPlain(username, password string) error { + key, err := precis.UsernameCaseMapped.CompareKey(username) + if err != nil { + return err + } + + hash, ok, err := a.table.Lookup(context.TODO(), key) + if !ok { + return module.ErrUnknownCredentials + } + if err != nil { + return err + } + + parts := strings.SplitN(hash, ":", 2) + if len(parts) != 2 { + return fmt.Errorf("%s: auth plain %s: no hash tag", a.modName, key) + } + hashVerify := HashVerify[parts[0]] + if hashVerify == nil { + return fmt.Errorf("%s: auth plain %s: unknown hash: %s", a.modName, key, parts[0]) + } + return hashVerify(password, parts[1]) +} + +func (a *Auth) ListUsers() ([]string, error) { + tbl, ok := a.table.(module.MutableTable) + if !ok { + return nil, fmt.Errorf("%s: table is not mutable, no management functionality available", a.modName) + } + + l, err := tbl.Keys() + if err != nil { + return nil, fmt.Errorf("%s: list users: %w", a.modName, err) + } + return l, nil +} + +func (a *Auth) CreateUser(username, password string) error { + return a.CreateUserHash(username, password, HashBcrypt, HashOpts{ + BcryptCost: bcrypt.DefaultCost, + }) +} + +func (a *Auth) CreateUserHash(username, password string, hashAlgo string, opts HashOpts) error { + tbl, ok := a.table.(module.MutableTable) + if !ok { + return fmt.Errorf("%s: table is not mutable, no management functionality available", a.modName) + } + + if _, ok := HashCompute[hashAlgo]; !ok { + return fmt.Errorf("%s: unknown hash function: %v", a.modName, hashAlgo) + } + + key, err := precis.UsernameCaseMapped.CompareKey(username) + if err != nil { + return fmt.Errorf("%s: create user %s (raw): %w", a.modName, username, err) + } + + _, ok, err = tbl.Lookup(context.TODO(), key) + if err != nil { + return fmt.Errorf("%s: create user %s: %w", a.modName, key, err) + } + if ok { + return fmt.Errorf("%s: credentials for %s already exist", a.modName, key) + } + + hash, err := HashCompute[hashAlgo](opts, password) + if err != nil { + return fmt.Errorf("%s: create user %s: hash generation: %w", a.modName, key, err) + } + + if err := tbl.SetKey(key, hashAlgo+":"+hash); err != nil { + return fmt.Errorf("%s: create user %s: %w", a.modName, key, err) + } + return nil +} + +func (a *Auth) SetUserPassword(username, password string) error { + tbl, ok := a.table.(module.MutableTable) + if !ok { + return fmt.Errorf("%s: table is not mutable, no management functionality available", a.modName) + } + + key, err := precis.UsernameCaseMapped.CompareKey(username) + if err != nil { + return fmt.Errorf("%s: set password %s (raw): %w", a.modName, username, err) + } + + // TODO: Allow to customize hash function. + hash, err := HashCompute[HashBcrypt](HashOpts{ + BcryptCost: bcrypt.DefaultCost, + }, password) + if err != nil { + return fmt.Errorf("%s: set password %s: hash generation: %w", a.modName, key, err) + } + + if err := tbl.SetKey(key, "bcrypt:"+hash); err != nil { + return fmt.Errorf("%s: set password %s: %w", a.modName, key, err) + } + return nil +} + +func (a *Auth) DeleteUser(username string) error { + tbl, ok := a.table.(module.MutableTable) + if !ok { + return fmt.Errorf("%s: table is not mutable, no management functionality available", a.modName) + } + + key, err := precis.UsernameCaseMapped.CompareKey(username) + if err != nil { + return fmt.Errorf("%s: del user %s (raw): %w", a.modName, username, err) + } + + if err := tbl.RemoveKey(key); err != nil { + return fmt.Errorf("%s: del user %s: %w", a.modName, key, err) + } + return nil +} + +func init() { + module.Register("auth.pass_table", New) +} diff --git a/internal/auth/pass_table/table_test.go b/internal/auth/pass_table/table_test.go new file mode 100644 index 0000000..666e2b6 --- /dev/null +++ b/internal/auth/pass_table/table_test.go @@ -0,0 +1,64 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package pass_table + +import ( + "testing" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestAuth_AuthPlain(t *testing.T) { + addSHA256() + + mod, err := New("pass_table", "", nil, []string{"dummy"}) + if err != nil { + t.Fatal(err) + } + err = mod.Init(config.NewMap(nil, config.Node{ + Children: []config.Node{}, + })) + if err != nil { + t.Fatal(err) + } + a := mod.(*Auth) + a.table = testutils.Table{ + M: map[string]string{ + "foxcpp": "sha256:U0FMVA==:8PDRAgaUqaLSk34WpYniXjaBgGM93Lc6iF4pw2slthw=", + "not-foxcpp": "bcrypt:$2y$10$4tEJtJ6dApmhETg8tJ4WHOeMtmYXQwmHDKIyfg09Bw1F/smhLjlaa", + "not-foxcpp-2": "argon2:1:8:1:U0FBQUFBTFQ=:KHUshl3DcpHR3AoVd28ZeBGmZ1Fj1gwJgNn98Ia8DAvGHqI0BvFOMJPxtaAfO8F+qomm2O3h0P0yV50QGwXI/Q==", + }, + } + + check := func(user, pass string, ok bool) { + t.Helper() + + err := a.AuthPlain(user, pass) + if (err == nil) != ok { + t.Errorf("ok=%v, err: %v", ok, err) + } + } + + check("foxcpp", "password", true) + check("foxcpp", "different-password", false) + check("not-foxcpp", "password", true) + check("not-foxcpp", "different-password", false) + check("not-foxcpp-2", "password", true) +} diff --git a/internal/auth/plain_separate/plain_separate.go b/internal/auth/plain_separate/plain_separate.go new file mode 100644 index 0000000..893d1f0 --- /dev/null +++ b/internal/auth/plain_separate/plain_separate.go @@ -0,0 +1,145 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package plain_separate + +import ( + "context" + "errors" + "fmt" + + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +type Auth struct { + modName string + instName string + + userTbls []module.Table + passwd []module.PlainAuth + + onlyFirstID bool + + Log log.Logger +} + +func NewAuth(modName, instName string, _, inlinargs []string) (module.Module, error) { + a := &Auth{ + modName: modName, + instName: instName, + onlyFirstID: false, + Log: log.Logger{Name: modName}, + } + + if len(inlinargs) != 0 { + return nil, errors.New("plain_separate: inline arguments are not used") + } + + return a, nil +} + +func (a *Auth) Name() string { + return a.modName +} + +func (a *Auth) InstanceName() string { + return a.instName +} + +func (a *Auth) Init(cfg *config.Map) error { + cfg.Bool("debug", false, false, &a.Log.Debug) + cfg.Callback("user", func(m *config.Map, node config.Node) error { + var tbl module.Table + err := modconfig.ModuleFromNode("table", node.Args, node, m.Globals, &tbl) + if err != nil { + return err + } + + a.userTbls = append(a.userTbls, tbl) + return nil + }) + cfg.Callback("pass", func(m *config.Map, node config.Node) error { + var auth module.PlainAuth + err := modconfig.ModuleFromNode("auth", node.Args, node, m.Globals, &auth) + if err != nil { + return err + } + + a.passwd = append(a.passwd, auth) + return nil + }) + + if _, err := cfg.Process(); err != nil { + return err + } + + return nil +} + +func (a *Auth) Lookup(ctx context.Context, username string) (string, bool, error) { + ok := len(a.userTbls) == 0 + for _, tbl := range a.userTbls { + _, tblOk, err := tbl.Lookup(ctx, username) + if err != nil { + return "", false, fmt.Errorf("plain_separate: underlying table error: %w", err) + } + if tblOk { + ok = true + break + } + } + if !ok { + return "", false, nil + } + return "", true, nil +} + +func (a *Auth) AuthPlain(username, password string) error { + ok := len(a.userTbls) == 0 + for _, tbl := range a.userTbls { + _, tblOk, err := tbl.Lookup(context.TODO(), username) + if err != nil { + return err + } + if tblOk { + ok = true + break + } + } + if !ok { + return errors.New("user not found in tables") + } + + var lastErr error + for _, p := range a.passwd { + if err := p.AuthPlain(username, password); err != nil { + lastErr = err + continue + } + + return nil + } + return lastErr +} + +func init() { + module.Register("auth.plain_separate", NewAuth) +} diff --git a/internal/auth/plain_separate/plain_separate_test.go b/internal/auth/plain_separate/plain_separate_test.go new file mode 100644 index 0000000..e5365cc --- /dev/null +++ b/internal/auth/plain_separate/plain_separate_test.go @@ -0,0 +1,155 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package plain_separate + +import ( + "context" + "errors" + "testing" + + "github.com/emersion/go-sasl" + "github.com/foxcpp/maddy/framework/module" +) + +type mockAuth struct { + db map[string]bool +} + +func (mockAuth) SASLMechanisms() []string { + return []string{sasl.Plain, sasl.Login} +} + +func (m mockAuth) AuthPlain(username, _ string) error { + ok := m.db[username] + if !ok { + return errors.New("invalid creds") + } + return nil +} + +type mockTable struct { + db map[string]string +} + +func (m mockTable) Lookup(_ context.Context, a string) (string, bool, error) { + b, ok := m.db[a] + return b, ok, nil +} + +func TestPlainSplit_NoUser(t *testing.T) { + a := Auth{ + passwd: []module.PlainAuth{ + mockAuth{ + db: map[string]bool{ + "user1": true, + }, + }, + }, + } + + err := a.AuthPlain("user1", "aaa") + if err != nil { + t.Fatal("Unexpected error:", err) + } +} + +func TestPlainSplit_NoUser_MultiPass(t *testing.T) { + a := Auth{ + passwd: []module.PlainAuth{ + mockAuth{ + db: map[string]bool{ + "user2": true, + }, + }, + mockAuth{ + db: map[string]bool{ + "user1": true, + }, + }, + }, + } + + err := a.AuthPlain("user1", "aaa") + if err != nil { + t.Fatal("Unexpected error:", err) + } +} + +func TestPlainSplit_UserPass(t *testing.T) { + a := Auth{ + userTbls: []module.Table{ + mockTable{ + db: map[string]string{ + "user1": "", + }, + }, + }, + passwd: []module.PlainAuth{ + mockAuth{ + db: map[string]bool{ + "user2": true, + }, + }, + mockAuth{ + db: map[string]bool{ + "user1": true, + }, + }, + }, + } + + err := a.AuthPlain("user1", "aaa") + if err != nil { + t.Fatal("Unexpected error:", err) + } +} + +func TestPlainSplit_MultiUser_Pass(t *testing.T) { + a := Auth{ + userTbls: []module.Table{ + mockTable{ + db: map[string]string{ + "userWH": "", + }, + }, + mockTable{ + db: map[string]string{ + "user1": "", + }, + }, + }, + passwd: []module.PlainAuth{ + mockAuth{ + db: map[string]bool{ + "user2": true, + }, + }, + mockAuth{ + db: map[string]bool{ + "user1": true, + }, + }, + }, + } + + err := a.AuthPlain("user1", "aaa") + if err != nil { + t.Fatal("Unexpected error:", err) + } +} diff --git a/internal/auth/sasl.go b/internal/auth/sasl.go new file mode 100644 index 0000000..8510052 --- /dev/null +++ b/internal/auth/sasl.go @@ -0,0 +1,207 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package auth + +import ( + "context" + "errors" + "fmt" + "net" + + "github.com/emersion/go-sasl" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth/sasllogin" + "github.com/foxcpp/maddy/internal/authz" +) + +var ( + ErrUnsupportedMech = errors.New("Unsupported SASL mechanism") + ErrInvalidAuthCred = errors.New("auth: invalid credentials") +) + +// SASLAuth is a wrapper that initializes sasl.Server using authenticators that +// call maddy module objects. +// +// It also handles username translation using auth_map and auth_map_normalize +// (AuthMap and AuthMapNormalize should be set). +// +// It supports reporting of multiple authorization identities so multiple +// accounts can be associated with a single set of credentials. +type SASLAuth struct { + Log log.Logger + OnlyFirstID bool + EnableLogin bool + + AuthMap module.Table + AuthNormalize authz.NormalizeFunc + + Plain []module.PlainAuth +} + +func (s *SASLAuth) SASLMechanisms() []string { + var mechs []string + + if len(s.Plain) != 0 { + mechs = append(mechs, sasl.Plain) + if s.EnableLogin { + mechs = append(mechs, sasl.Login) + } + } + + return mechs +} + +func (s *SASLAuth) usernameForAuth(ctx context.Context, saslUsername string) (string, error) { + if s.AuthNormalize != nil { + var err error + saslUsername, err = s.AuthNormalize(saslUsername) + if err != nil { + return "", err + } + } + + if s.AuthMap == nil { + return saslUsername, nil + } + + mapped, ok, err := s.AuthMap.Lookup(ctx, saslUsername) + if err != nil { + return "", err + } + if !ok { + return "", ErrInvalidAuthCred + } + + if saslUsername != mapped { + s.Log.DebugMsg("using mapped username for authentication", "username", saslUsername, "mapped_username", mapped) + } + + return mapped, nil +} + +func (s *SASLAuth) AuthPlain(username, password string) error { + if len(s.Plain) == 0 { + return ErrUnsupportedMech + } + + var lastErr error + for _, p := range s.Plain { + mappedUsername, err := s.usernameForAuth(context.TODO(), username) + if err != nil { + return err + } + + s.Log.DebugMsg("attempting authentication", + "mapped_username", mappedUsername, "original_username", username, + "module", p) + + lastErr = p.AuthPlain(mappedUsername, password) + if lastErr == nil { + return nil + } + } + + return fmt.Errorf("no auth. provider accepted creds, last err: %w", lastErr) +} + +type ContextData struct { + // Authentication username. May be different from identity. + Username string + + // Password used for password-based mechanisms. + Password string +} + +// CreateSASL creates the sasl.Server instance for the corresponding mechanism. +func (s *SASLAuth) CreateSASL(mech string, remoteAddr net.Addr, successCb func(identity string, data ContextData) error) sasl.Server { + switch mech { + case sasl.Plain: + return sasl.NewPlainServer(func(identity, username, password string) error { + if identity == "" { + identity = username + } + if identity != username { + return ErrInvalidAuthCred + } + + err := s.AuthPlain(username, password) + if err != nil { + s.Log.Error("authentication failed", err, "username", username, "src_ip", remoteAddr) + return ErrInvalidAuthCred + } + + return successCb(identity, ContextData{ + Username: username, + Password: password, + }) + }) + case sasl.Login: + if !s.EnableLogin { + return FailingSASLServ{Err: ErrUnsupportedMech} + } + + return sasllogin.NewLoginServer(func(username, password string) error { + username, err := s.usernameForAuth(context.Background(), username) + if err != nil { + return err + } + + err = s.AuthPlain(username, password) + if err != nil { + s.Log.Error("authentication failed", err, "username", username, "src_ip", remoteAddr) + return ErrInvalidAuthCred + } + + return successCb(username, ContextData{ + Username: username, + Password: password, + }) + }) + } + return FailingSASLServ{Err: ErrUnsupportedMech} +} + +// AddProvider adds the SASL authentication provider to its mapping by parsing +// the 'auth' configuration directive. +func (s *SASLAuth) AddProvider(m *config.Map, node config.Node) error { + var any interface{} + if err := modconfig.ModuleFromNode("auth", node.Args, node, m.Globals, &any); err != nil { + return err + } + + hasAny := false + if plainAuth, ok := any.(module.PlainAuth); ok { + s.Plain = append(s.Plain, plainAuth) + hasAny = true + } + + if !hasAny { + return config.NodeErr(node, "auth: specified module does not provide any SASL mechanism") + } + return nil +} + +type FailingSASLServ struct{ Err error } + +func (s FailingSASLServ) Next([]byte) ([]byte, bool, error) { + return nil, true, s.Err +} diff --git a/internal/auth/sasl_test.go b/internal/auth/sasl_test.go new file mode 100644 index 0000000..a59cfc7 --- /dev/null +++ b/internal/auth/sasl_test.go @@ -0,0 +1,89 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package auth + +import ( + "errors" + "net" + "testing" + + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +type mockAuth struct { + db map[string]bool +} + +func (m mockAuth) AuthPlain(username, _ string) error { + ok := m.db[username] + if !ok { + return errors.New("invalid creds") + } + return nil +} + +func TestCreateSASL(t *testing.T) { + a := SASLAuth{ + Log: testutils.Logger(t, "saslauth"), + Plain: []module.PlainAuth{ + &mockAuth{ + db: map[string]bool{ + "user1": true, + }, + }, + }, + } + + t.Run("XWHATEVER", func(t *testing.T) { + srv := a.CreateSASL("XWHATEVER", &net.TCPAddr{}, func(string, ContextData) error { return nil }) + _, _, err := srv.Next([]byte("")) + if err == nil { + t.Error("No error for XWHATEVER use") + } + }) + + t.Run("PLAIN", func(t *testing.T) { + srv := a.CreateSASL("PLAIN", &net.TCPAddr{}, func(id string, data ContextData) error { + if id != "user1" { + t.Fatal("Wrong auth. identities passed to callback:", id) + } + return nil + }) + + _, _, err := srv.Next([]byte("\x00user1\x00aa")) + if err != nil { + t.Error("Unexpected error:", err) + } + }) + + t.Run("PLAIN with authorization identity", func(t *testing.T) { + srv := a.CreateSASL("PLAIN", &net.TCPAddr{}, func(id string, data ContextData) error { + if id != "user1" { + t.Fatal("Wrong authorization identity passed:", id) + } + return nil + }) + + _, _, err := srv.Next([]byte("user1\x00user1\x00aa")) + if err != nil { + t.Error("Unexpected error:", err) + } + }) +} diff --git a/internal/auth/sasllogin/sasllogin.go b/internal/auth/sasllogin/sasllogin.go new file mode 100644 index 0000000..fac5026 --- /dev/null +++ b/internal/auth/sasllogin/sasllogin.go @@ -0,0 +1,54 @@ +package sasllogin + +import "github.com/emersion/go-sasl" + +// Copy-pasted from old emersion/go-sasl version + +// Authenticates users with an username and a password. +type LoginAuthenticator func(username, password string) error +type loginState int + +const ( + loginNotStarted loginState = iota + loginWaitingUsername + loginWaitingPassword +) + +type loginServer struct { + state loginState + username, password string + authenticate LoginAuthenticator +} + +// A server implementation of the LOGIN authentication mechanism, as described +// in https://tools.ietf.org/html/draft-murchison-sasl-login-00. +// +// LOGIN is obsolete and should only be enabled for legacy clients that cannot +// be updated to use PLAIN. +func NewLoginServer(authenticator LoginAuthenticator) sasl.Server { + return &loginServer{authenticate: authenticator} +} + +func (a *loginServer) Next(response []byte) (challenge []byte, done bool, err error) { + switch a.state { + case loginNotStarted: + // Check for initial response field, as per RFC4422 section 3 + if response == nil { + challenge = []byte("Username:") + break + } + a.state++ + fallthrough + case loginWaitingUsername: + a.username = string(response) + challenge = []byte("Password:") + case loginWaitingPassword: + a.password = string(response) + err = a.authenticate(a.username, a.password) + done = true + default: + err = sasl.ErrUnexpectedClientResponse + } + a.state++ + return +} diff --git a/internal/auth/shadow/module.go b/internal/auth/shadow/module.go new file mode 100644 index 0000000..92307bf --- /dev/null +++ b/internal/auth/shadow/module.go @@ -0,0 +1,138 @@ +//go:build !windows +// +build !windows + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package shadow + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth/external" +) + +type Auth struct { + instName string + useHelper bool + helperPath string + + Log log.Logger +} + +func New(modName, instName string, _, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, errors.New("shadow: inline arguments are not used") + } + return &Auth{ + instName: instName, + Log: log.Logger{Name: modName}, + }, nil +} + +func (a *Auth) Name() string { + return "shadow" +} + +func (a *Auth) InstanceName() string { + return a.instName +} + +func (a *Auth) Init(cfg *config.Map) error { + cfg.Bool("debug", true, false, &a.Log.Debug) + cfg.Bool("use_helper", false, false, &a.useHelper) + if _, err := cfg.Process(); err != nil { + return err + } + + if a.useHelper { + a.helperPath = filepath.Join(config.LibexecDirectory, "maddy-shadow-helper") + if _, err := os.Stat(a.helperPath); err != nil { + return fmt.Errorf("shadow: no helper binary (maddy-shadow-helper) found in %s", config.LibexecDirectory) + } + } else { + f, err := os.Open("/etc/shadow") + if err != nil { + if os.IsPermission(err) { + return fmt.Errorf("shadow: can't read /etc/shadow due to permission error, use helper binary or run maddy as a privileged user") + } + return fmt.Errorf("shadow: can't read /etc/shadow: %v", err) + } + f.Close() + } + + return nil +} + +func (a *Auth) Lookup(username string) (string, bool, error) { + if a.useHelper { + return "", false, fmt.Errorf("shadow: table lookup are not possible when using a helper") + } + + ent, err := Lookup(username) + if err != nil { + if errors.Is(err, ErrNoSuchUser) { + return "", false, nil + } + return "", false, err + } + + if !ent.IsAccountValid() { + return "", false, nil + } + + return "", true, nil +} + +func (a *Auth) AuthPlain(username, password string) error { + if a.useHelper { + return external.AuthUsingHelper(a.helperPath, username, password) + } + + ent, err := Lookup(username) + if err != nil { + return err + } + + if !ent.IsAccountValid() { + return fmt.Errorf("shadow: account is expired") + } + + if !ent.IsPasswordValid() { + return fmt.Errorf("shadow: password is expired") + } + + if err := ent.VerifyPassword(password); err != nil { + if errors.Is(err, ErrWrongPassword) { + return module.ErrUnknownCredentials + } + return err + } + + return nil +} + +func init() { + module.Register("auth.shadow", New) +} diff --git a/internal/auth/shadow/read.go b/internal/auth/shadow/read.go new file mode 100644 index 0000000..7059aa0 --- /dev/null +++ b/internal/auth/shadow/read.go @@ -0,0 +1,100 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package shadow + +import ( + "bufio" + "errors" + "fmt" + "os" + "strconv" + "strings" +) + +var ( + ErrNoSuchUser = errors.New("shadow: user entry is not present in database") + ErrWrongPassword = errors.New("shadow: wrong password") +) + +// Read reads system shadow passwords database and returns all entires in it. +func Read() ([]Entry, error) { + f, err := os.Open("/etc/shadow") + if err != nil { + return nil, err + } + scnr := bufio.NewScanner(f) + + var res []Entry + for scnr.Scan() { + ent, err := parseEntry(scnr.Text()) + if err != nil { + return res, err + } + + res = append(res, *ent) + } + if err := scnr.Err(); err != nil { + return res, err + } + return res, nil +} + +func parseEntry(line string) (*Entry, error) { + parts := strings.Split(line, ":") + if len(parts) != 9 { + return nil, errors.New("read: malformed entry") + } + + res := &Entry{ + Name: parts[0], + Pass: parts[1], + } + + for i, value := range [...]*int{ + &res.LastChange, &res.MinPassAge, &res.MaxPassAge, + &res.WarnPeriod, &res.InactivityPeriod, &res.AcctExpiry, &res.Flags, + } { + if parts[2+i] == "" { + *value = -1 + } else { + var err error + *value, err = strconv.Atoi(parts[2+i]) + if err != nil { + return nil, fmt.Errorf("read: invalid value for field %d", 2+i) + } + } + } + + return res, nil +} + +func Lookup(name string) (*Entry, error) { + entries, err := Read() + if err != nil { + return nil, err + } + + for _, entry := range entries { + if entry.Name == name { + return &entry, nil + } + } + + return nil, ErrNoSuchUser +} diff --git a/internal/auth/shadow/shadow.go b/internal/auth/shadow/shadow.go new file mode 100644 index 0000000..eddf6ee --- /dev/null +++ b/internal/auth/shadow/shadow.go @@ -0,0 +1,64 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// shadow package implements utilities for parsing and using shadow password +// database on Unix systems. +package shadow + +type Entry struct { + // User login name. + Name string + + // Hashed user password. + Pass string + + // Days since Jan 1, 1970 password was last changed. + LastChange int + + // The number of days the user will have to wait before she will be allowed to + // change her password again. + // + // -1 if password aging is disabled. + MinPassAge int + + // The number of days after which the user will have to change her password. + // + // -1 is password aging is disabled. + MaxPassAge int + + // The number of days before a password is going to expire (see the maximum + // password age above) during which the user should be warned. + // + // -1 is password aging is disabled. + WarnPeriod int + + // The number of days after a password has expired (see the maximum + // password age above) during which the password should still be accepted. + // + // -1 is password aging is disabled. + InactivityPeriod int + + // The date of expiration of the account, expressed as the number of days + // since Jan 1, 1970. + // + // -1 is account never expires. + AcctExpiry int + + // Unused now. + Flags int +} diff --git a/internal/auth/shadow/verify.go b/internal/auth/shadow/verify.go new file mode 100644 index 0000000..7c983d5 --- /dev/null +++ b/internal/auth/shadow/verify.go @@ -0,0 +1,74 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package shadow + +import ( + "errors" + "fmt" + "time" + + "github.com/GehirnInc/crypt" + _ "github.com/GehirnInc/crypt/sha256_crypt" + _ "github.com/GehirnInc/crypt/sha512_crypt" +) + +const secsInDay = 86400 + +func (e *Entry) IsAccountValid() bool { + if e.AcctExpiry == -1 { + return true + } + + nowDays := int(time.Now().Unix() / secsInDay) + return nowDays < e.AcctExpiry +} + +func (e *Entry) IsPasswordValid() bool { + if e.LastChange == -1 || e.MaxPassAge == -1 || e.InactivityPeriod == -1 { + return true + } + + nowDays := int(time.Now().Unix() / secsInDay) + return nowDays < e.LastChange+e.MaxPassAge+e.InactivityPeriod +} + +func (e *Entry) VerifyPassword(pass string) (err error) { + // Do not permit null and locked passwords. + if e.Pass == "" { + return errors.New("verify: null password") + } + if e.Pass[0] == '!' { + return errors.New("verify: locked password") + } + + // crypt.NewFromHash may panic on unknown hash function. + defer func() { + if rcvr := recover(); rcvr != nil { + err = fmt.Errorf("%v", rcvr) + } + }() + + if err := crypt.NewFromHash(e.Pass).Verify(e.Pass, []byte(pass)); err != nil { + if errors.Is(err, crypt.ErrKeyMismatch) { + return ErrWrongPassword + } + return err + } + return nil +} diff --git a/internal/authz/lookup.go b/internal/authz/lookup.go new file mode 100644 index 0000000..f19c3f8 --- /dev/null +++ b/internal/authz/lookup.go @@ -0,0 +1,44 @@ +package authz + +import ( + "context" + "fmt" + + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/module" +) + +func AuthorizeEmailUse(ctx context.Context, username string, addrs []string, mapping module.Table) (bool, error) { + var validEmails []string + + if multi, ok := mapping.(module.MultiTable); ok { + var err error + validEmails, err = multi.LookupMulti(ctx, username) + if err != nil { + return false, fmt.Errorf("authz: %w", err) + } + } else { + validEmail, ok, err := mapping.Lookup(ctx, username) + if err != nil { + return false, fmt.Errorf("authz: %w", err) + } + if ok { + validEmails = []string{validEmail} + } + } + + for _, addr := range addrs { + _, domain, err := address.Split(addr) + if err != nil { + return false, fmt.Errorf("authz: %w", err) + } + + for _, ent := range validEmails { + if ent == domain || ent == "*" || ent == addr { + return true, nil + } + } + } + + return false, nil +} diff --git a/internal/authz/normalization.go b/internal/authz/normalization.go new file mode 100644 index 0000000..99c46d0 --- /dev/null +++ b/internal/authz/normalization.go @@ -0,0 +1,37 @@ +package authz + +import ( + "strings" + + "github.com/foxcpp/maddy/framework/address" + "golang.org/x/text/secure/precis" +) + +type NormalizeFunc func(string) (string, error) + +func NormalizeNoop(s string) (string, error) { + return s, nil +} + +// NormalizeAuto applies address.PRECISFold to valid emails and +// plain UsernameCaseMapped profile to other strings. +func NormalizeAuto(s string) (string, error) { + if address.Valid(s) { + return address.PRECISFold(s) + } + return precis.UsernameCaseMapped.CompareKey(s) +} + +// NormalizeFuncs defines configurable normalization functions to be used +// in authentication and authorization routines. +var NormalizeFuncs = map[string]NormalizeFunc{ + "auto": NormalizeAuto, + "precis_casefold_email": address.PRECISFold, + "precis_casefold": precis.UsernameCaseMapped.CompareKey, + "precis_email": address.PRECIS, + "precis": precis.UsernameCasePreserved.CompareKey, + "casefold": func(s string) (string, error) { + return strings.ToLower(s), nil + }, + "noop": NormalizeNoop, +} diff --git a/internal/check/authorize_sender/authorize_sender.go b/internal/check/authorize_sender/authorize_sender.go new file mode 100644 index 0000000..6827add --- /dev/null +++ b/internal/check/authorize_sender/authorize_sender.go @@ -0,0 +1,305 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package authorize_sender + +import ( + "context" + "net/mail" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/authz" + "github.com/foxcpp/maddy/internal/table" + "github.com/foxcpp/maddy/internal/target" +) + +const modName = "check.authorize_sender" + +type Check struct { + instName string + log log.Logger + + checkHeader bool + emailPrepare module.Table + userToEmail module.Table + + unauthAction modconfig.FailAction + noMatchAction modconfig.FailAction + errAction modconfig.FailAction + + fromNorm authz.NormalizeFunc + authNorm authz.NormalizeFunc +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + return &Check{ + instName: instName, + }, nil +} + +func (c *Check) Name() string { + return modName +} + +func (c *Check) InstanceName() string { + return c.instName +} + +func (c *Check) Init(cfg *config.Map) error { + cfg.Bool("debug", true, false, &c.log.Debug) + + cfg.Bool("check_header", false, true, &c.checkHeader) + + cfg.Custom("prepare_email", false, false, func() (interface{}, error) { + return &table.Identity{}, nil + }, modconfig.TableDirective, &c.emailPrepare) + cfg.Custom("user_to_email", false, false, func() (interface{}, error) { + return &table.Identity{}, nil + }, modconfig.TableDirective, &c.userToEmail) + + cfg.Custom("unauth_action", false, false, func() (interface{}, error) { + return modconfig.FailAction{Reject: true}, nil + }, modconfig.FailActionDirective, &c.unauthAction) + cfg.Custom("no_match_action", false, false, func() (interface{}, error) { + return modconfig.FailAction{Reject: true}, nil + }, modconfig.FailActionDirective, &c.noMatchAction) + cfg.Custom("err_action", false, false, func() (interface{}, error) { + return modconfig.FailAction{Reject: true}, nil + }, modconfig.FailActionDirective, &c.errAction) + + config.EnumMapped(cfg, "auth_normalize", true, false, authz.NormalizeFuncs, authz.NormalizeAuto, + &c.authNorm) + config.EnumMapped(cfg, "from_normalize", true, false, authz.NormalizeFuncs, authz.NormalizeAuto, + &c.fromNorm) + + if _, err := cfg.Process(); err != nil { + return err + } + + return nil +} + +type state struct { + c *Check + msgMeta *module.MsgMetadata + log log.Logger +} + +func (c *Check) CheckStateForMsg(_ context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &state{ + c: c, + msgMeta: msgMeta, + log: target.DeliveryLogger(c.log, msgMeta), + }, nil +} + +func (s *state) authzSender(ctx context.Context, authName, email string) module.CheckResult { + if authName == "" { + return s.c.unauthAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 530, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Authentication required", + CheckName: modName, + }}) + } + + fromEmailNorm, err := s.c.fromNorm(email) + if err != nil { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 1, 7}, + Message: "Unable to normalize sender address", + CheckName: modName, + Err: err, + }}) + } + authNameNorm, err := s.c.authNorm(authName) + if err != nil { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 535, + EnhancedCode: exterrors.EnhancedCode{5, 7, 8}, + Message: "Unable to normalize authorization username", + CheckName: modName, + }}) + } + + var preparedEmail []string + var ok bool + s.log.DebugMsg("normalized names", "from", fromEmailNorm, "auth", authNameNorm) + if emailPrepareMulti, isMulti := s.c.emailPrepare.(module.MultiTable); isMulti { + preparedEmail, err = emailPrepareMulti.LookupMulti(ctx, fromEmailNorm) + ok = len(preparedEmail) > 0 + } else { + var preparedEmail_single string + preparedEmail_single, ok, err = s.c.emailPrepare.Lookup(ctx, fromEmailNorm) + preparedEmail = []string{preparedEmail_single} + } + s.log.DebugMsg("authorized emails", "preparedEmail", preparedEmail, "ok", ok) + if err != nil { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 454, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Internal error during policy check", + CheckName: modName, + Err: err, + }}) + } + if !ok { + preparedEmail = []string{fromEmailNorm} + } + + ok, err = authz.AuthorizeEmailUse(ctx, authNameNorm, preparedEmail, s.c.userToEmail) + if err != nil { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 454, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Internal error during policy check", + CheckName: modName, + Err: err, + }}) + } + if !ok { + return s.c.noMatchAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Unauthorized use of sender address", + CheckName: modName, + }}) + } + + return module.CheckResult{} +} + +func (s *state) CheckConnection(_ context.Context) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckSender(ctx context.Context, fromEmail string) module.CheckResult { + if s.msgMeta.Conn == nil { + s.log.Msg("skipping locally generated message") + return module.CheckResult{} + } + authName := s.msgMeta.Conn.AuthUser + + return s.authzSender(ctx, authName, fromEmail) +} + +func (s *state) CheckRcpt(_ context.Context, _ string) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckBody(ctx context.Context, hdr textproto.Header, _ buffer.Buffer) module.CheckResult { + if !s.c.checkHeader { + return module.CheckResult{} + } + if s.msgMeta.Conn == nil { + s.log.Msg("skipping locally generated message") + return module.CheckResult{} + } + authName := s.msgMeta.Conn.AuthUser + + fromHdr := hdr.Get("From") + if fromHdr == "" { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Missing From header", + CheckName: modName, + }}) + } + list, err := mail.ParseAddressList(fromHdr) + if err != nil || len(list) == 0 { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Malformed From header", + CheckName: modName, + Err: err, + }}) + } + fromEmail := list[0].Address + if len(list) > 1 { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Multiple From addresses are not allowed", + CheckName: modName, + Err: err, + }}) + } + + var senderAddr string + if senderHdr := hdr.Get("Sender"); senderHdr != "" { + sender, err := mail.ParseAddress(senderHdr) + if err != nil { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Malformed Sender header", + CheckName: modName, + Err: err, + }}) + } + senderAddr = sender.Address + } + + res := s.authzSender(ctx, authName, fromEmail) + if res.Reason == nil { + return res + } + + if senderAddr != "" && senderAddr != fromEmail { + res = s.authzSender(ctx, authName, senderAddr) + if res.Reason == nil { + return res + } + } + + // Neither matched. + return s.c.noMatchAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Unauthorized use of sender address", + CheckName: modName, + }}) +} + +func (s *state) Close() error { + return nil +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/check/command/command.go b/internal/check/command/command.go new file mode 100644 index 0000000..6093760 --- /dev/null +++ b/internal/check/command/command.go @@ -0,0 +1,401 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package command + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "regexp" + "runtime/trace" + "strconv" + "strings" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +const modName = "check.command" + +type Stage string + +const ( + StageConnection = "conn" + StageSender = "sender" + StageRcpt = "rcpt" + StageBody = "body" +) + +var placeholderRe = regexp.MustCompile(`{[a-zA-Z0-9_]+?}`) + +type Check struct { + instName string + log log.Logger + + stage Stage + actions map[int]modconfig.FailAction + cmd string + cmdArgs []string +} + +func New(modName, instName string, aliases, inlineArgs []string) (module.Module, error) { + c := &Check{ + instName: instName, + actions: map[int]modconfig.FailAction{ + 1: { + Reject: true, + }, + 2: { + Quarantine: true, + }, + }, + } + + if len(inlineArgs) == 0 { + return nil, errors.New("command: at least one argument is required (command name)") + } + + c.cmd = inlineArgs[0] + c.cmdArgs = inlineArgs[1:] + + return c, nil +} + +func (c *Check) Name() string { + return modName +} + +func (c *Check) InstanceName() string { + return c.instName +} + +func (c *Check) Init(cfg *config.Map) error { + // Check whether the inline argument command is usable. + if _, err := exec.LookPath(c.cmd); err != nil { + return fmt.Errorf("command: %w", err) + } + + cfg.Enum("run_on", false, false, + []string{StageConnection, StageSender, StageRcpt, StageBody}, StageBody, + (*string)(&c.stage)) + + cfg.AllowUnknown() + unknown, err := cfg.Process() + if err != nil { + return err + } + + for _, node := range unknown { + switch node.Name { + case "code": + if len(node.Args) < 2 { + return config.NodeErr(node, "at least two arguments are required: ") + } + exitCode, err := strconv.Atoi(node.Args[0]) + if err != nil { + return config.NodeErr(node, "%v", err) + } + action, err := modconfig.ParseActionDirective(node.Args[1:]) + if err != nil { + return config.NodeErr(node, "%v", err) + } + + c.actions[exitCode] = action + default: + return config.NodeErr(node, "unexpected directive: %v", node.Name) + } + } + + return nil +} + +type state struct { + c *Check + msgMeta *module.MsgMetadata + log log.Logger + + mailFrom string + rcpts []string +} + +func (c *Check) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &state{ + c: c, + msgMeta: msgMeta, + log: target.DeliveryLogger(c.log, msgMeta), + }, nil +} + +func (s *state) expandCommand(address string) (string, []string) { + expArgs := make([]string, len(s.c.cmdArgs)) + + for i, arg := range s.c.cmdArgs { + expArgs[i] = placeholderRe.ReplaceAllStringFunc(arg, func(placeholder string) string { + switch placeholder { + case "{auth_user}": + if s.msgMeta.Conn == nil { + return "" + } + return s.msgMeta.Conn.AuthUser + case "{source_ip}": + if s.msgMeta.Conn == nil { + return "" + } + tcpAddr, _ := s.msgMeta.Conn.RemoteAddr.(*net.TCPAddr) + if tcpAddr == nil { + return "" + } + return tcpAddr.IP.String() + case "{source_host}": + if s.msgMeta.Conn == nil { + return "" + } + return s.msgMeta.Conn.Hostname + case "{source_rdns}": + if s.msgMeta.Conn == nil { + return "" + } + valI, err := s.msgMeta.Conn.RDNSName.Get() + if err != nil { + return "" + } + if valI == nil { + return "" + } + return valI.(string) + case "{msg_id}": + return s.msgMeta.ID + case "{sender}": + return s.mailFrom + case "{rcpts}": + return strings.Join(s.rcpts, "\n") + case "{address}": + return address + } + return placeholder + }) + } + + return s.c.cmd, expArgs +} + +func (s *state) run(cmdName string, args []string, stdin io.Reader) module.CheckResult { + cmd := exec.Command(cmdName, args...) + cmd.Stdin = stdin + stdout, err := cmd.StdoutPipe() + if err != nil { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 450, + Message: "Internal server error", + CheckName: "command", + Err: err, + Misc: map[string]interface{}{ + "cmd": cmd.String(), + }, + }, + Reject: true, + } + } + + if err := cmd.Start(); err != nil { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 450, + Message: "Internal server error", + CheckName: "command", + Err: err, + Misc: map[string]interface{}{ + "cmd": cmd.String(), + }, + }, + Reject: true, + } + } + + bufOut := bufio.NewReader(stdout) + hdr, err := textproto.ReadHeader(bufOut) + if err != nil && !errors.Is(err, io.EOF) { + if err := cmd.Process.Signal(os.Interrupt); err != nil { + s.log.Error("failed to kill process", err) + } + + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 450, + Message: "Internal server error", + CheckName: "command", + Err: err, + Misc: map[string]interface{}{ + "cmd": cmd.String(), + }, + }, + Reject: true, + } + } + + res := module.CheckResult{} + res.Header = hdr + + err = cmd.Wait() + if err != nil { + if _, ok := err.(*exec.ExitError); !ok { + // If that's not ExitError, the process may still be running. We do + // not want this. + if err := cmd.Process.Signal(os.Interrupt); err != nil { + s.log.Error("failed to kill process", err) + } + } + return s.errorRes(err, res, cmd.String()) + } + return res +} + +func (s *state) errorRes(err error, res module.CheckResult, cmdLine string) module.CheckResult { + exitErr, ok := err.(*exec.ExitError) + if !ok { + res.Reason = &exterrors.SMTPError{ + Code: 450, + Message: "Internal server error", + CheckName: "command", + Err: err, + Misc: map[string]interface{}{ + "cmd": cmdLine, + }, + } + res.Reject = true + return res + } + + action, ok := s.c.actions[exitErr.ExitCode()] + if !ok { + res.Reason = &exterrors.SMTPError{ + Code: 450, + Message: "Internal server error", + CheckName: "command", + Err: err, + Reason: "unexpected exit code", + Misc: map[string]interface{}{ + "cmd": cmdLine, + "exit_code": exitErr.ExitCode(), + }, + } + res.Reject = true + return res + } + + res.Reason = &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 1}, + Message: "Message rejected for due to a local policy", + CheckName: "command", + Misc: map[string]interface{}{ + "cmd": cmdLine, + "exit_code": exitErr.ExitCode(), + }, + } + + return action.Apply(res) +} + +func (s *state) CheckConnection(ctx context.Context) module.CheckResult { + if s.c.stage != StageConnection { + return module.CheckResult{} + } + + defer trace.StartRegion(ctx, "command/CheckConnection-"+s.c.cmd).End() + + cmdName, cmdArgs := s.expandCommand("") + return s.run(cmdName, cmdArgs, bytes.NewReader(nil)) +} + +func (s *state) CheckSender(ctx context.Context, addr string) module.CheckResult { + s.mailFrom = addr + + if s.c.stage != StageSender { + return module.CheckResult{} + } + + defer trace.StartRegion(ctx, "command/CheckSender"+s.c.cmd).End() + + cmdName, cmdArgs := s.expandCommand(addr) + return s.run(cmdName, cmdArgs, bytes.NewReader(nil)) +} + +func (s *state) CheckRcpt(ctx context.Context, addr string) module.CheckResult { + s.rcpts = append(s.rcpts, addr) + + if s.c.stage != StageRcpt { + return module.CheckResult{} + } + defer trace.StartRegion(ctx, "command/CheckRcpt"+s.c.cmd).End() + + cmdName, cmdArgs := s.expandCommand(addr) + return s.run(cmdName, cmdArgs, bytes.NewReader(nil)) +} + +func (s *state) CheckBody(ctx context.Context, hdr textproto.Header, body buffer.Buffer) module.CheckResult { + if s.c.stage != StageBody { + return module.CheckResult{} + } + + defer trace.StartRegion(ctx, "command/CheckBody"+s.c.cmd).End() + + cmdName, cmdArgs := s.expandCommand("") + + var buf bytes.Buffer + _ = textproto.WriteHeader(&buf, hdr) + bR, err := body.Open() + if err != nil { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 450, + Message: "Internal server error", + CheckName: "command", + Err: err, + Misc: map[string]interface{}{ + "cmd": cmdName + " " + strings.Join(cmdArgs, " "), + }, + }, + Reject: true, + } + } + + return s.run(cmdName, cmdArgs, io.MultiReader(bytes.NewReader(buf.Bytes()), bR)) +} + +func (s *state) Close() error { + return nil +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/check/dkim/dkim.go b/internal/check/dkim/dkim.go new file mode 100644 index 0000000..563fc5b --- /dev/null +++ b/internal/check/dkim/dkim.go @@ -0,0 +1,272 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dkim + +import ( + "bytes" + "context" + "errors" + "io" + nettextproto "net/textproto" + "runtime/trace" + "strings" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/emersion/go-msgauth/dkim" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +type Check struct { + instName string + log log.Logger + + requiredFields map[string]struct{} + brokenSigAction modconfig.FailAction + noSigAction modconfig.FailAction + failOpen bool + + resolver dns.Resolver +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, errors.New("check.dkim: inline arguments are not used") + } + return &Check{ + instName: instName, + log: log.Logger{Name: "check.dkim"}, + resolver: dns.DefaultResolver(), + }, nil +} + +func (c *Check) Init(cfg *config.Map) error { + var requiredFields []string + + cfg.Bool("debug", true, false, &c.log.Debug) + cfg.StringList("required_fields", false, false, []string{"From", "Subject"}, &requiredFields) + cfg.Bool("fail_open", false, false, &c.failOpen) + cfg.Custom("broken_sig_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.brokenSigAction) + cfg.Custom("no_sig_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.noSigAction) + _, err := cfg.Process() + if err != nil { + return err + } + + c.requiredFields = make(map[string]struct{}) + for _, field := range requiredFields { + c.requiredFields[nettextproto.CanonicalMIMEHeaderKey(field)] = struct{}{} + } + + return nil +} + +func (c *Check) Name() string { + return "check.dkim" +} + +func (c *Check) InstanceName() string { + return c.instName +} + +type dkimCheckState struct { + c *Check + msgMeta *module.MsgMetadata + log log.Logger +} + +func (d *dkimCheckState) CheckConnection(ctx context.Context) module.CheckResult { + return module.CheckResult{} +} + +func (d *dkimCheckState) CheckSender(ctx context.Context, mailFrom string) module.CheckResult { + return module.CheckResult{} +} + +func (d *dkimCheckState) CheckRcpt(ctx context.Context, rcptTo string) module.CheckResult { + return module.CheckResult{} +} + +func (d *dkimCheckState) CheckBody(ctx context.Context, header textproto.Header, body buffer.Buffer) module.CheckResult { + defer trace.StartRegion(ctx, "check.dkim/CheckBody").End() + + if !header.Has("DKIM-Signature") { + if d.c.noSigAction.Reject || d.c.noSigAction.Quarantine { + d.log.Printf("no signatures present") + } else { + d.log.Debugf("no signatures present") + } + return d.c.noSigAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 20}, + Message: "No DKIM signatures", + CheckName: "check.dkim", + }, + AuthResult: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultNone, + }, + }, + }) + } + + b := bytes.Buffer{} + _ = textproto.WriteHeader(&b, header) + bodyRdr, err := body.Open() + if err != nil { + return module.CheckResult{ + Reject: true, + Reason: exterrors.WithTemporary( + exterrors.WithFields(err, map[string]interface{}{ + "check": "check.dkim", + "smtp_msg": "Internal I/O error", + }), + true, + ), + } + } + + verifications, err := dkim.VerifyWithOptions(io.MultiReader(&b, bodyRdr), &dkim.VerifyOptions{ + LookupTXT: func(domain string) ([]string, error) { + return d.c.resolver.LookupTXT(ctx, domain) + }, + }) + if err != nil { + return module.CheckResult{ + Reject: true, + Reason: exterrors.WithTemporary( + exterrors.WithFields(err, map[string]interface{}{ + "check": "check.dkim", + "smtp_msg": "Internal error during policy check", + }), + true, + ), + } + } + + goodSigs := false + + res := module.CheckResult{AuthResult: make([]authres.Result, 0, len(verifications))} + for _, verif := range verifications { + val := authres.ResultValue(authres.ResultPass) + reason := "" + if verif.Err != nil { + val = authres.ResultFail + + reason = strings.TrimPrefix(verif.Err.Error(), "dkim: ") + if !d.c.brokenSigAction.Reject || !d.c.brokenSigAction.Quarantine { + d.log.DebugMsg("bad signature", "domain", verif.Domain, "identifier", verif.Identifier) + } + if dkim.IsPermFail(verif.Err) { + val = authres.ResultPermError + } + if dkim.IsTempFail(verif.Err) { + if !d.c.failOpen { + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 421, + EnhancedCode: exterrors.EnhancedCode{4, 7, 20}, + Message: "Temporary error during DKIM verification", + CheckName: "check.dkim", + Err: verif.Err, + }, + } + } + val = authres.ResultTempError + } + + res.AuthResult = append(res.AuthResult, &authres.DKIMResult{ + Value: val, + Reason: reason, + Domain: verif.Domain, + Identifier: verif.Identifier, + }) + continue + } + + signedFields := make(map[string]struct{}, len(verif.HeaderKeys)) + for _, field := range verif.HeaderKeys { + signedFields[nettextproto.CanonicalMIMEHeaderKey(field)] = struct{}{} + } + for field := range d.c.requiredFields { + if _, ok := signedFields[field]; !ok { + val = authres.ResultPermError + reason = "some header fields are not signed" + } + } + + if val == authres.ResultPass { + goodSigs = true + d.log.DebugMsg("good signature", "domain", verif.Domain, "identifier", verif.Identifier) + } + + res.AuthResult = append(res.AuthResult, &authres.DKIMResult{ + Value: val, + Reason: reason, + Domain: verif.Domain, + Identifier: verif.Identifier, + }) + } + + if !goodSigs { + res.Reason = &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 20}, + Message: "No passing DKIM signatures", + CheckName: "check.dkim", + } + return d.c.brokenSigAction.Apply(res) + } + return res +} + +func (d *dkimCheckState) Name() string { + return "check.dkim" +} + +func (d *dkimCheckState) Close() error { + return nil +} + +func (c *Check) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &dkimCheckState{ + c: c, + msgMeta: msgMeta, + log: target.DeliveryLogger(c.log, msgMeta), + }, nil +} + +func init() { + module.Register("check.dkim", New) +} diff --git a/internal/check/dkim/dkim_test.go b/internal/check/dkim/dkim_test.go new file mode 100644 index 0000000..020d50f --- /dev/null +++ b/internal/check/dkim/dkim_test.go @@ -0,0 +1,364 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dkim + +import ( + "context" + "errors" + "net" + "testing" + + "github.com/emersion/go-msgauth/authres" + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +const unsignedMailString = `From: Joe SixPack +To: Suzie Q +Subject: Is dinner ready? +Date: Fri, 11 Jul 2003 21:00:37 -0700 (PDT) +Message-ID: <20030712040037.46341.5F8J@football.example.com> + +Hi. + +We lost the game. Are you hungry yet? + +Joe. +` + +const dnsPublicKey = "v=DKIM1; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQ" + + "KBgQDwIRP/UC3SBsEmGqZ9ZJW3/DkMoGeLnQg1fWn7/zYt" + + "IxN2SnFCjxOCKG9v3b4jYfcTNh5ijSsq631uBItLa7od+v" + + "/RtdC2UzJ1lWT947qR+Rcac2gbto/NMqJ0fzfVjH4OuKhi" + + "tdY9tf6mcwGjaNBcWToIMmPSPDdQPNUYckcQ2QIDAQAB" + +var testZones = map[string]mockdns.Zone{ + "brisbane._domainkey.example.com.": { + TXT: []string{dnsPublicKey}, + }, +} + +const verifiedMailString = `DKIM-Signature: v=1; a=rsa-sha256; s=brisbane; d=example.com; + c=simple/simple; q=dns/txt; i=joe@football.example.com; + h=Received : From : To : Subject : Date : Message-ID; + bh=2jUSOH9NhtVGCQWNr9BrIAPreKQjO6Sn7XIkfJVOzv8=; + b=AuUoFEfDxTDkHlLXSZEpZj79LICEps6eda7W3deTVFOk4yAUoqOB + 4nujc7YopdG5dWLSdNg6xNAZpOPr+kHxt1IrE+NahM6L/LbvaHut + KVdkLLkpVaVVQPzeRDI009SO2Il5Lu7rDNH6mZckBdrIx0orEtZV + 4bmp/YzhwvcubU4=; +Received: from client1.football.example.com [192.0.2.1] + by submitserver.example.com with SUBMISSION; + Fri, 11 Jul 2003 21:01:54 -0700 (PDT) +From: Joe SixPack +To: Suzie Q +Subject: Is dinner ready? +Date: Fri, 11 Jul 2003 21:00:37 -0700 (PDT) +Message-ID: <20030712040037.46341.5F8J@football.example.com> + +Hi. + +We lost the game. Are you hungry yet? + +Joe. +` + +func testCheck(t *testing.T, zones map[string]mockdns.Zone, cfg []config.Node) *Check { + t.Helper() + mod, err := New("check.dkim", "", nil, nil) + if err != nil { + t.Fatal(err) + } + check := mod.(*Check) + check.resolver = &mockdns.Resolver{Zones: zones} + check.log = testutils.Logger(t, mod.Name()) + + if err := check.Init(config.NewMap(nil, config.Node{Children: cfg})); err != nil { + t.Fatal(err) + } + + return check +} + +func TestDkimVerify_NoSig(t *testing.T) { + check := testCheck(t, nil, nil) // No zones since this test requires no lookups. + + // Force certain reason so we can assert for it. + check.noSigAction.Reject = true + check.noSigAction.ReasonOverride = &exterrors.SMTPError{Code: 555} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // The usual checking flow. + s, err := check.CheckStateForMsg(ctx, &module.MsgMetadata{ + ID: "test_unsigned", + }) + if err != nil { + t.Fatal(err) + } + s.CheckConnection(ctx) + s.CheckSender(ctx, "joe@football.example.com") + s.CheckRcpt(ctx, "suzie@shopping.example.net") + + hdr, buf := testutils.BodyFromStr(t, unsignedMailString) + result := s.CheckBody(ctx, hdr, buf) + + if result.Reason == nil { + t.Fatal("No check fail reason set, auth. result:", authres.Format("", result.AuthResult)) + } + if result.Reason.(*exterrors.SMTPError).Code != 555 { + t.Fatal("Different fail reason:", result.Reason) + } +} + +func TestDkimVerify_InvalidSig(t *testing.T) { + check := testCheck(t, testZones, nil) + + // Force certain reason so we can assert for it. + check.brokenSigAction.Reject = true + check.brokenSigAction.ReasonOverride = &exterrors.SMTPError{Code: 555} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := check.CheckStateForMsg(ctx, &module.MsgMetadata{ + ID: "test_unsigned", + }) + if err != nil { + t.Fatal(err) + } + + s.CheckConnection(ctx) + s.CheckSender(ctx, "joe@football.example.com") + s.CheckRcpt(ctx, "suzie@shopping.example.net") + + hdr, buf := testutils.BodyFromStr(t, verifiedMailString) + // Mess up the signature. + hdr.Set("From", "nope") + + result := s.CheckBody(ctx, hdr, buf) + + if result.Reason == nil { + t.Fatal("No check fail reason set, auth. result:", authres.Format("", result.AuthResult)) + } + if result.Reason.(*exterrors.SMTPError).Code != 555 { + t.Fatal("Different fail reason:", result.Reason) + } +} + +func TestDkimVerify_ValidSig(t *testing.T) { + check := testCheck(t, testZones, nil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := check.CheckStateForMsg(ctx, &module.MsgMetadata{ + ID: "test_unsigned", + }) + if err != nil { + t.Fatal(err) + } + + s.CheckConnection(ctx) + s.CheckSender(ctx, "joe@football.example.com") + s.CheckRcpt(ctx, "suzie@shopping.example.net") + + hdr, buf := testutils.BodyFromStr(t, verifiedMailString) + + result := s.CheckBody(ctx, hdr, buf) + + if result.Reason != nil { + t.Log(authres.Format("", result.AuthResult)) + t.Fatal("Check fail reason set, auth. result:", result.Reason, exterrors.Fields(result.Reason)) + } +} + +func TestDkimVerify_RequiredFields(t *testing.T) { + check := testCheck(t, testZones, []config.Node{ + { + // Require field that is not covered by the signature. + Name: "required_fields", + Args: []string{"From", "X-Important"}, + }, + }) + + // Force certain reason so we can assert for it. + check.brokenSigAction.Reject = true + check.brokenSigAction.ReasonOverride = &exterrors.SMTPError{Code: 555} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := check.CheckStateForMsg(ctx, &module.MsgMetadata{ + ID: "test_unsigned", + }) + if err != nil { + t.Fatal(err) + } + + s.CheckConnection(ctx) + s.CheckSender(ctx, "joe@football.example.com") + s.CheckRcpt(ctx, "suzie@shopping.example.net") + + hdr, buf := testutils.BodyFromStr(t, verifiedMailString) + + result := s.CheckBody(ctx, hdr, buf) + + if result.Reason == nil { + t.Fatal("No check fail reason set, auth. result:", authres.Format("", result.AuthResult)) + } + if result.Reason.(*exterrors.SMTPError).Code != 555 { + t.Fatal("Different fail reason:", result.Reason) + } +} + +func TestDkimVerify_BufferOpenFail(t *testing.T) { + check := testCheck(t, testZones, nil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := check.CheckStateForMsg(ctx, &module.MsgMetadata{ + ID: "test_unsigned", + }) + if err != nil { + t.Fatal(err) + } + + s.CheckConnection(ctx) + s.CheckSender(ctx, "joe@football.example.com") + s.CheckRcpt(ctx, "suzie@shopping.example.net") + + var buf buffer.Buffer + hdr, buf := testutils.BodyFromStr(t, verifiedMailString) + buf = testutils.FailingBuffer{Blob: buf.(buffer.MemoryBuffer).Slice, OpenError: errors.New("No!")} + + result := s.CheckBody(ctx, hdr, buf) + t.Log("auth. result:", authres.Format("", result.AuthResult)) + + if result.Reason == nil { + t.Fatal("No check fail reason set, auth. result:", authres.Format("", result.AuthResult)) + } +} + +func TestDkimVerify_FailClosed(t *testing.T) { + zones := map[string]mockdns.Zone{ + "brisbane._domainkey.example.com.": { + Err: &net.DNSError{ + Err: "DNS server is not having a great time", + IsTemporary: true, + IsTimeout: true, + }, + }, + } + check := testCheck(t, zones, []config.Node{ + { + Name: "fail_open", + Args: []string{"false"}, + }, + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := check.CheckStateForMsg(ctx, &module.MsgMetadata{ + ID: "test_unsigned", + }) + if err != nil { + t.Fatal(err) + } + + s.CheckConnection(ctx) + s.CheckSender(ctx, "joe@football.example.com") + s.CheckRcpt(ctx, "suzie@shopping.example.net") + + hdr, buf := testutils.BodyFromStr(t, verifiedMailString) + + result := s.CheckBody(ctx, hdr, buf) + t.Log("auth. result:", authres.Format("", result.AuthResult)) + + if result.Reason == nil { + t.Fatal("No check fail reason set, auth. result:", authres.Format("", result.AuthResult)) + } + if !result.Reject { + t.Fatal("No reject requested") + } + if !exterrors.IsTemporary(result.Reason) { + t.Fatal("Fail reason is not marked as temporary:", result.Reason) + } +} + +func TestDkimVerify_FailOpen(t *testing.T) { + zones := map[string]mockdns.Zone{ + "brisbane._domainkey.example.com.": { + Err: &net.DNSError{ + Err: "DNS server is not having a great time", + IsTemporary: true, + IsTimeout: true, + }, + }, + } + check := testCheck(t, zones, []config.Node{ + { + Name: "fail_open", + Args: []string{"true"}, + }, + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := check.CheckStateForMsg(ctx, &module.MsgMetadata{ + ID: "test_unsigned", + }) + if err != nil { + t.Fatal(err) + } + + s.CheckConnection(ctx) + s.CheckSender(ctx, "joe@football.example.com") + s.CheckRcpt(ctx, "suzie@shopping.example.net") + + hdr, buf := testutils.BodyFromStr(t, verifiedMailString) + + result := s.CheckBody(ctx, hdr, buf) + + t.Log("auth. result:", authres.Format("", result.AuthResult)) + if result.Reason == nil { + t.Fatal("No check fail reason set, auth. result:", authres.Format("", result.AuthResult)) + } + if result.Reject { + t.Fatal("Reject requested") + } + if exterrors.IsTemporary(result.Reason) { + t.Fatal("Fail reason is not marked as temporary:", result.Reason) + } + + if len(result.AuthResult) != 1 { + t.Fatal("Wrong amount of auth. result fields:", len(result.AuthResult)) + } + resVal := result.AuthResult[0].(*authres.DKIMResult).Value + if resVal != authres.ResultTempError { + t.Fatal("Result is not temp. error:", resVal) + } +} diff --git a/internal/check/dns/dns.go b/internal/check/dns/dns.go new file mode 100644 index 0000000..80c89ed --- /dev/null +++ b/internal/check/dns/dns.go @@ -0,0 +1,163 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dns + +import ( + "strings" + + "github.com/foxcpp/maddy/framework/address" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/check" +) + +func requireMatchingRDNS(ctx check.StatelessCheckContext) module.CheckResult { + if ctx.MsgMeta.Conn == nil { + ctx.Logger.Msg("locally-generated message, skipping") + return module.CheckResult{} + } + if ctx.MsgMeta.Conn.RDNSName == nil { + ctx.Logger.Msg("rDNS lookup is disabled, skipping") + return module.CheckResult{} + } + + rdnsNameI, err := ctx.MsgMeta.Conn.RDNSName.Get() + if err != nil { + reason, misc := exterrors.UnwrapDNSErr(err) + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: exterrors.SMTPCode(err, 450, 550), + EnhancedCode: exterrors.SMTPEnchCode(err, exterrors.EnhancedCode{0, 7, 25}), + Message: "DNS error during policy check", + CheckName: "require_matching_rdns", + Err: err, + Reason: reason, + Misc: misc, + }, + } + } + if rdnsNameI == nil { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 25}, + Message: "No PTR record found", + CheckName: "require_matching_rdns", + Err: err, + }, + } + } + rdnsName := rdnsNameI.(string) + + srcDomain := strings.TrimSuffix(ctx.MsgMeta.Conn.Hostname, ".") + rdnsName = strings.TrimSuffix(rdnsName, ".") + + if dns.Equal(rdnsName, srcDomain) { + ctx.Logger.Debugf("PTR record %s matches source domain, OK", rdnsName) + return module.CheckResult{} + } + + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 25}, + Message: "rDNS name does not match source hostname", + CheckName: "require_matching_rdns", + }, + } +} + +func requireMXRecord(ctx check.StatelessCheckContext, mailFrom string) module.CheckResult { + if mailFrom == "" { + // Permit null reverse-path for bounces. + return module.CheckResult{} + } + + _, domain, err := address.Split(mailFrom) + if err != nil { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 8}, + Message: "Malformed sender address", + CheckName: "require_mx_record", + }, + } + } + if domain == "" { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 8}, + Message: "No domain part in address", + CheckName: "require_mx_record", + }, + } + } + + srcMx, err := ctx.Resolver.LookupMX(ctx, domain) + if err != nil { + reason, misc := exterrors.UnwrapDNSErr(err) + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: exterrors.SMTPCode(err, 450, 550), + EnhancedCode: exterrors.SMTPEnchCode(err, exterrors.EnhancedCode{0, 7, 0}), + Message: "DNS error during policy check", + CheckName: "require_mx_record", + Err: err, + Reason: reason, + Misc: misc, + }, + } + } + + if len(srcMx) == 0 { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 7, 27}, + Message: "Domain in MAIL FROM does not have any MX records", + CheckName: "require_mx_record", + }, + } + } + + for _, mx := range srcMx { + if mx.Host == "." { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 7, 27}, + Message: "Domain in MAIL FROM has null MX record", + CheckName: "require_mx_record", + }, + } + } + } + + return module.CheckResult{} +} +func init() { + check.RegisterStatelessCheck("require_matching_rdns", modconfig.FailAction{Quarantine: true}, + requireMatchingRDNS, nil, nil, nil) + check.RegisterStatelessCheck("require_mx_record", modconfig.FailAction{Quarantine: true}, + nil, requireMXRecord, nil, nil) +} diff --git a/internal/check/dns/dns_test.go b/internal/check/dns/dns_test.go new file mode 100644 index 0000000..ca6c348 --- /dev/null +++ b/internal/check/dns/dns_test.go @@ -0,0 +1,116 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dns + +import ( + "net" + "testing" + + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/framework/future" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/check" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestRequireMatchingRDNS(t *testing.T) { + test := func(rdns, srcHost string, fail bool) { + rdnsFut := future.New() + var ptr []string + if rdns != "" { + rdnsFut.Set(rdns, nil) + ptr = []string{rdns} + } else { + rdnsFut.Set(nil, nil) + } + + res := requireMatchingRDNS(check.StatelessCheckContext{ + Resolver: &mockdns.Resolver{ + Zones: map[string]mockdns.Zone{ + "4.3.2.1.in-addr.arpa.": { + PTR: ptr, + }, + }, + }, + MsgMeta: &module.MsgMetadata{ + Conn: &module.ConnState{ + RemoteAddr: &net.TCPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 55555}, + Hostname: srcHost, + RDNSName: rdnsFut, + }, + }, + Logger: testutils.Logger(t, "require_matching_rdns"), + }) + + actualFail := res.Reason != nil + if fail && !actualFail { + t.Errorf("%v, %s: expected failure but check succeeded", rdns, srcHost) + } + if !fail && actualFail { + t.Errorf("%v, %s: unexpected failure", rdns, srcHost) + } + } + + test("", "example.org", true) + test("example.org", "[1.2.3.4]", true) + test("example.org", "[IPv6:beef::1]", true) + test("example.org", "example.org", false) + test("example.org.", "example.org", false) + test("example.org", "example.org.", false) + test("example.org.", "example.org.", false) + test("example.com.", "example.org.", true) +} + +func TestRequireMXRecord(t *testing.T) { + test := func(mailFrom, mxDomain string, mx []net.MX, fail bool) { + res := requireMXRecord(check.StatelessCheckContext{ + Resolver: &mockdns.Resolver{ + Zones: map[string]mockdns.Zone{ + mxDomain + ".": { + MX: mx, + }, + }, + }, + MsgMeta: &module.MsgMetadata{ + Conn: &module.ConnState{ + RemoteAddr: &net.TCPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 55555}, + }, + }, + Logger: testutils.Logger(t, "require_mx_record"), + }, mailFrom) + + actualFail := res.Reason != nil + if fail && !actualFail { + t.Errorf("%v, %v: expected failure but check succeeded", mailFrom, mx) + } + if !fail && actualFail { + t.Errorf("%v, %v: unexpected failure", mailFrom, mx) + } + } + + test("foo@example.org", "example.org", nil, true) + test("foo@example.com", "", nil, true) // NXDOMAIN + test("foo@[1.2.3.4]", "", nil, true) + test("[IPv6:beef::1]", "", nil, true) + test("[IPv6:beef::1]", "", nil, true) + test("foo@example.org", "example.org", []net.MX{{Host: "a.com"}}, false) + test("foo@", "", nil, true) + test("", "", nil, false) // Permit <> for bounces. + test("foo@example.org", "example.org", []net.MX{{Host: "."}}, true) +} diff --git a/internal/check/dnsbl/common.go b/internal/check/dnsbl/common.go new file mode 100644 index 0000000..7b874cb --- /dev/null +++ b/internal/check/dnsbl/common.go @@ -0,0 +1,197 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dnsbl + +import ( + "context" + "net" + "strconv" + "strings" + + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" +) + +type ListedErr struct { + Identity string + List string + Reason string +} + +func (le ListedErr) Fields() map[string]interface{} { + return map[string]interface{}{ + "check": "dnsbl", + "list": le.List, + "listed_identity": le.Identity, + "reason": le.Reason, + "smtp_code": 554, + "smtp_enchcode": exterrors.EnhancedCode{5, 7, 0}, + "smtp_msg": "Client identity listed in the used DNSBL", + } +} + +func (le ListedErr) Error() string { + return le.Identity + " is listed in the used DNSBL" +} + +func checkDomain(ctx context.Context, resolver dns.Resolver, cfg List, domain string) error { + query := domain + "." + cfg.Zone + + addrs, err := resolver.LookupHost(ctx, query) + if err != nil { + if dnsErr, ok := err.(*net.DNSError); ok && dnsErr.IsNotFound { + return nil + } + + return err + } + + if len(addrs) == 0 { + return nil + } + + // Attempt to extract explanation string. + txts, err := resolver.LookupTXT(context.Background(), query) + if err != nil || len(txts) == 0 { + // Not significant, include addresses as reason. Usually they are + // mapped to some predefined 'reasons' by BL. + return ListedErr{ + Identity: domain, + List: cfg.Zone, + Reason: strings.Join(addrs, "; "), + } + } + + // Some BLs provide multiple reasons (meta-BLs such as Spamhaus Zen) so + // don't mangle them by joining with "", instead join with "; ". + + return ListedErr{ + Identity: domain, + List: cfg.Zone, + Reason: strings.Join(txts, "; "), + } +} + +func checkIP(ctx context.Context, resolver dns.Resolver, cfg List, ip net.IP) error { + ipv6 := true + if ipv4 := ip.To4(); ipv4 != nil { + ip = ipv4 + ipv6 = false + } + + if ipv6 && !cfg.ClientIPv6 { + return nil + } + if !ipv6 && !cfg.ClientIPv4 { + return nil + } + + query := queryString(ip) + "." + cfg.Zone + + addrs, err := resolver.LookupIPAddr(ctx, query) + if err != nil { + if dnsErr, ok := err.(*net.DNSError); ok && dnsErr.IsNotFound { + return nil + } + + return err + } + + filteredAddrs := make([]net.IPAddr, 0, len(addrs)) +addrsLoop: + for _, addr := range addrs { + // No responses whitelist configured - permit all. + if len(cfg.Responses) == 0 { + filteredAddrs = append(filteredAddrs, addr) + continue + } + + for _, respNet := range cfg.Responses { + if respNet.Contains(addr.IP) { + filteredAddrs = append(filteredAddrs, addr) + continue addrsLoop + } + } + } + + if len(filteredAddrs) == 0 { + return nil + } + + // Attempt to extract explanation string. + txts, err := resolver.LookupTXT(ctx, query) + if err != nil || len(txts) == 0 { + // Not significant, include addresses as reason. Usually they are + // mapped to some predefined 'reasons' by BL. + + reasonParts := make([]string, 0, len(filteredAddrs)) + for _, addr := range filteredAddrs { + reasonParts = append(reasonParts, addr.IP.String()) + } + + return ListedErr{ + Identity: ip.String(), + List: cfg.Zone, + Reason: strings.Join(reasonParts, "; "), + } + } + + // Some BLs provide multiple reasons (meta-BLs such as Spamhaus Zen) so + // don't mangle them by joining with "", instead join with "; ". + + return ListedErr{ + Identity: ip.String(), + List: cfg.Zone, + Reason: strings.Join(txts, "; "), + } +} + +func queryString(ip net.IP) string { + ipv6 := true + if ipv4 := ip.To4(); ipv4 != nil { + ip = ipv4 + ipv6 = false + } + + res := strings.Builder{} + if ipv6 { + res.Grow(63) // 0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 + } else { + res.Grow(15) // 000.000.000.000 + } + + for i := len(ip) - 1; i >= 0; i-- { + octet := ip[i] + + if ipv6 { + // X.X + res.WriteString(strconv.FormatInt(int64(octet&0xf), 16)) + res.WriteRune('.') + res.WriteString(strconv.FormatInt(int64((octet&0xf0)>>4), 16)) + } else { + // X + res.WriteString(strconv.Itoa(int(octet))) + } + + if i != 0 { + res.WriteRune('.') + } + } + return res.String() +} diff --git a/internal/check/dnsbl/common_test.go b/internal/check/dnsbl/common_test.go new file mode 100644 index 0000000..caa0657 --- /dev/null +++ b/internal/check/dnsbl/common_test.go @@ -0,0 +1,238 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dnsbl + +import ( + "context" + "net" + "reflect" + "testing" + + "github.com/foxcpp/go-mockdns" +) + +func TestQueryString(t *testing.T) { + test := func(ip, queryStr string) { + t.Helper() + + parsed := net.ParseIP(ip) + if parsed == nil { + panic("Malformed IP in test") + } + + actual := queryString(parsed) + if actual != queryStr { + t.Errorf("want queryString(%s) to be %s, got %s", ip, queryStr, actual) + } + } + + test("2001:db8:1:2:3:4:567:89ab", "b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.2.0.0.0.1.0.0.0.8.b.d.0.1.0.0.2") + test("2001::1:2:3:4:567:89ab", "b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.2.0.0.0.1.0.0.0.0.0.0.0.1.0.0.2") + test("192.0.2.99", "99.2.0.192") +} + +func TestCheckDomain(t *testing.T) { + test := func(zones map[string]mockdns.Zone, cfg List, domain string, expectedErr error) { + t.Helper() + resolver := mockdns.Resolver{Zones: zones} + err := checkDomain(context.Background(), &resolver, cfg, domain) + if !reflect.DeepEqual(err, expectedErr) { + t.Errorf("expected err to be '%#v', got '%#v'", expectedErr, err) + } + } + + test(nil, List{Zone: "example.org"}, "example.com", nil) + test(map[string]mockdns.Zone{ + "example.com.example.org.": { + Err: &net.DNSError{ + Err: "i/o timeout", + IsTimeout: true, + IsTemporary: true, + }, + }, + }, List{Zone: "example.org"}, "example.com", &net.DNSError{ + Err: "i/o timeout", + IsTimeout: true, + IsTemporary: true, + }) + test(map[string]mockdns.Zone{ + "example.com.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org"}, "example.com", ListedErr{ + Identity: "example.com", + List: "example.org", + Reason: "127.0.0.1", + }) + test(map[string]mockdns.Zone{ + "example.org.example.com.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org"}, "example.com", nil) + test(map[string]mockdns.Zone{ + "example.com.example.org.": { + A: []string{"127.0.0.1"}, + TXT: []string{"Reason"}, + }, + }, List{Zone: "example.org"}, "example.com", ListedErr{ + Identity: "example.com", + List: "example.org", + Reason: "Reason", + }) + test(map[string]mockdns.Zone{ + "example.com.example.org.": { + A: []string{"127.0.0.1"}, + TXT: []string{"Reason 1", "Reason 2"}, + }, + }, List{Zone: "example.org"}, "example.com", ListedErr{ + Identity: "example.com", + List: "example.org", + Reason: "Reason 1; Reason 2", + }) + test(map[string]mockdns.Zone{ + "example.com.example.org.": { + A: []string{"127.0.0.1", "127.0.0.2"}, + }, + }, List{Zone: "example.org"}, "example.com", ListedErr{ + Identity: "example.com", + List: "example.org", + Reason: "127.0.0.1; 127.0.0.2", + }) +} + +func TestCheckIP(t *testing.T) { + test := func(zones map[string]mockdns.Zone, cfg List, ip net.IP, expectedErr error) { + t.Helper() + resolver := mockdns.Resolver{Zones: zones} + err := checkIP(context.Background(), &resolver, cfg, ip) + if !reflect.DeepEqual(err, expectedErr) { + t.Errorf("expected err to be '%#v', got '%#v'", expectedErr, err) + } + } + + test(nil, List{Zone: "example.org"}, net.IPv4(1, 2, 3, 4), nil) + test(nil, List{Zone: "example.org", ClientIPv4: true}, net.IPv4(1, 2, 3, 4), nil) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", ClientIPv4: true}, net.IPv4(1, 2, 3, 4), ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Reason: "127.0.0.1", + }) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"128.0.0.1"}, + }, + }, List{ + Zone: "example.org", + ClientIPv4: true, + Responses: []net.IPNet{ + { + IP: net.IPv4(127, 0, 0, 1), + Mask: net.IPv4Mask(255, 255, 255, 0), + }, + }, + }, net.IPv4(1, 2, 3, 4), nil) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"128.0.0.1"}, + }, + }, List{ + Zone: "example.org", + ClientIPv4: true, + Responses: []net.IPNet{ + { + IP: net.IPv4(127, 0, 0, 0), + Mask: net.IPv4Mask(255, 255, 255, 0), + }, + { + IP: net.IPv4(128, 0, 0, 0), + Mask: net.IPv4Mask(255, 255, 255, 0), + }, + }, + }, net.IPv4(1, 2, 3, 4), ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Reason: "128.0.0.1", + }) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org"}, net.IPv4(1, 2, 3, 4), nil) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + Err: &net.DNSError{ + Err: "i/o timeout", + IsTimeout: true, + IsTemporary: true, + }, + }, + }, List{Zone: "example.org", ClientIPv4: true}, net.IPv4(1, 2, 3, 4), &net.DNSError{ + Err: "i/o timeout", + IsTimeout: true, + IsTemporary: true, + }) + + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + TXT: []string{"Reason"}, + }, + }, List{Zone: "example.org", ClientIPv4: true}, net.IPv4(1, 2, 3, 4), ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Reason: "Reason", + }) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1", "127.0.0.2"}, + }, + }, List{Zone: "example.org", ClientIPv4: true}, net.IPv4(1, 2, 3, 4), ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Reason: "127.0.0.1; 127.0.0.2", + }) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1", "127.0.0.2"}, + TXT: []string{"Reason", "Reason 2"}, + }, + }, List{Zone: "example.org", ClientIPv4: true}, net.IPv4(1, 2, 3, 4), ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Reason: "Reason; Reason 2", + }) + test(map[string]mockdns.Zone{ + "b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.2.0.0.0.1.0.0.0.8.b.d.0.1.0.0.2.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", ClientIPv4: true}, net.ParseIP("2001:db8:1:2:3:4:567:89ab"), nil) + test(map[string]mockdns.Zone{ + "b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.2.0.0.0.1.0.0.0.8.b.d.0.1.0.0.2.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", ClientIPv6: true}, net.ParseIP("2001:db8:1:2:3:4:567:89ab"), ListedErr{ + Identity: "2001:db8:1:2:3:4:567:89ab", + List: "example.org", + Reason: "127.0.0.1", + }) +} diff --git a/internal/check/dnsbl/dnsbl.go b/internal/check/dnsbl/dnsbl.go new file mode 100644 index 0000000..2c91c83 --- /dev/null +++ b/internal/check/dnsbl/dnsbl.go @@ -0,0 +1,435 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dnsbl + +import ( + "context" + "errors" + "net" + "runtime/trace" + "strings" + "sync" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" + "golang.org/x/sync/errgroup" +) + +type List struct { + Zone string + + ClientIPv4 bool + ClientIPv6 bool + + EHLO bool + MAILFROM bool + + ScoreAdj int + Responses []net.IPNet +} + +var defaultBL = List{ + ClientIPv4: true, +} + +type DNSBL struct { + instName string + checkEarly bool + inlineBls []string + bls []List + + quarantineThres int + rejectThres int + + resolver dns.Resolver + log log.Logger +} + +func NewDNSBL(_, instName string, _, inlineArgs []string) (module.Module, error) { + return &DNSBL{ + instName: instName, + inlineBls: inlineArgs, + + resolver: dns.DefaultResolver(), + log: log.Logger{Name: "dnsbl"}, + }, nil +} + +func (bl *DNSBL) Name() string { + return "dnsbl" +} + +func (bl *DNSBL) InstanceName() string { + return bl.instName +} + +func (bl *DNSBL) Init(cfg *config.Map) error { + cfg.Bool("debug", false, false, &bl.log.Debug) + cfg.Bool("check_early", false, false, &bl.checkEarly) + cfg.Int("quarantine_threshold", false, false, 1, &bl.quarantineThres) + cfg.Int("reject_threshold", false, false, 9999, &bl.rejectThres) + cfg.AllowUnknown() + unknown, err := cfg.Process() + if err != nil { + return err + } + + for _, inlineBl := range bl.inlineBls { + cfg := defaultBL + cfg.Zone = inlineBl + go bl.testList(cfg) + bl.bls = append(bl.bls, cfg) + } + + for _, node := range unknown { + if err := bl.readListCfg(node); err != nil { + return err + } + } + + return nil +} + +func (bl *DNSBL) readListCfg(node config.Node) error { + var ( + listCfg List + responseNets []string + ) + + cfg := config.NewMap(nil, node) + cfg.Bool("client_ipv4", false, defaultBL.ClientIPv4, &listCfg.ClientIPv4) + cfg.Bool("client_ipv6", false, defaultBL.ClientIPv4, &listCfg.ClientIPv6) + cfg.Bool("ehlo", false, defaultBL.EHLO, &listCfg.EHLO) + cfg.Bool("mailfrom", false, defaultBL.EHLO, &listCfg.MAILFROM) + cfg.Int("score", false, false, 1, &listCfg.ScoreAdj) + cfg.StringList("responses", false, false, []string{"127.0.0.1/24"}, &responseNets) + if _, err := cfg.Process(); err != nil { + return err + } + + for _, resp := range responseNets { + // If there is no / - it is a plain IP address, append + // '/32'. + if !strings.Contains(resp, "/") { + resp += "/32" + } + + _, ipNet, err := net.ParseCIDR(resp) + if err != nil { + return err + } + listCfg.Responses = append(listCfg.Responses, *ipNet) + } + + for _, zone := range append([]string{node.Name}, node.Args...) { + zoneCfg := listCfg + zoneCfg.Zone = zone + + if listCfg.ScoreAdj < 0 { + if zoneCfg.EHLO { + return errors.New("dnsbl: 'ehlo' should not be used with negative score") + } + if zoneCfg.MAILFROM { + return errors.New("dnsbl: 'mailfrom' should not be used with negative score") + } + } + bl.bls = append(bl.bls, zoneCfg) + + // From RFC 5782 Section 7: + // >To avoid this situation, systems that use + // >DNSxLs SHOULD check for the test entries described in Section 5 to + // >ensure that a domain actually has the structure of a DNSxL, and + // >SHOULD NOT use any DNSxL domain that does not have correct test + // >entries. + // Sadly, however, many DNSBLs lack test records so at most we can + // log a warning. Also, DNS is kinda slow so we do checks + // asynchronously to prevent slowing down server start-up. + go bl.testList(zoneCfg) + } + + return nil +} + +func (bl *DNSBL) testList(listCfg List) { + // Check RFC 5782 Section 5 requirements. + + bl.log.DebugMsg("testing list for RFC 5782 requirements...", "list", listCfg.Zone) + + // 1. IPv4-based DNSxLs MUST contain an entry for 127.0.0.2 for testing purposes. + if listCfg.ClientIPv4 { + err := checkIP(context.Background(), bl.resolver, listCfg, net.IPv4(127, 0, 0, 2)) + if err == nil { + bl.log.Msg("List does not contain a test record for 127.0.0.2", "list", listCfg.Zone) + } else if _, ok := err.(ListedErr); !ok { + bl.log.Error("lookup error, bailing out", err, "list", listCfg.Zone) + return + } + + // 2. IPv4-based DNSxLs MUST NOT contain an entry for 127.0.0.1. + err = checkIP(context.Background(), bl.resolver, listCfg, net.IPv4(127, 0, 0, 1)) + if err != nil { + _, ok := err.(ListedErr) + if !ok { + bl.log.Error("lookup error, bailing out", err, "list", listCfg.Zone) + return + } + bl.log.Msg("List contains a record for 127.0.0.1", "list", listCfg.Zone) + } + } + + if listCfg.ClientIPv6 { + // 1. IPv6-based DNSxLs MUST contain an entry for ::FFFF:7F00:2 + mustIP := net.ParseIP("::FFFF:7F00:2") + + err := checkIP(context.Background(), bl.resolver, listCfg, mustIP) + if err == nil { + bl.log.Msg("List does not contain a test record for ::FFFF:7F00:2", "list", listCfg.Zone) + } else if _, ok := err.(ListedErr); !ok { + bl.log.Error("lookup error, bailing out", err, "list", listCfg.Zone) + return + } + + // 2. IPv4-based DNSxLs MUST NOT contain an entry for ::FFFF:7F00:1 + mustNotIP := net.ParseIP("::FFFF:7F00:1") + err = checkIP(context.Background(), bl.resolver, listCfg, mustNotIP) + if err != nil { + _, ok := err.(ListedErr) + if !ok { + bl.log.Error("lookup error, bailing out", err, "list", listCfg.Zone) + return + } + bl.log.Msg("List contains a record for ::FFFF:7F00:1", "list", listCfg.Zone) + } + } + + if listCfg.EHLO || listCfg.MAILFROM { + // Domain-name-based DNSxLs MUST contain an entry for the reserved + // domain name "TEST". + err := checkDomain(context.Background(), bl.resolver, listCfg, "test") + if err == nil { + bl.log.Msg("List does not contain a test record for 'test' TLD", "list", listCfg.Zone) + } else if _, ok := err.(ListedErr); !ok { + bl.log.Error("lookup error, bailing out", err, "list", listCfg.Zone) + return + } + + // ... and MUST NOT contain an entry for the reserved domain name + // "INVALID". + err = checkDomain(context.Background(), bl.resolver, listCfg, "invalid") + if err != nil { + _, ok := err.(ListedErr) + if !ok { + bl.log.Error("lookup error, bailing out", err, "list", listCfg.Zone) + return + } + bl.log.Msg("List contains a record for 'invalid' TLD", "list", listCfg.Zone) + } + } +} + +func (bl *DNSBL) checkList(ctx context.Context, list List, ip net.IP, ehlo, mailFrom string) error { + if list.ClientIPv4 || list.ClientIPv6 { + if err := checkIP(ctx, bl.resolver, list, ip); err != nil { + return err + } + } + + if list.EHLO && ehlo != "" { + // Skip IPs in EHLO. + if strings.HasPrefix(ehlo, "[") && strings.HasSuffix(ehlo, "]") { + return nil + } + + if err := checkDomain(ctx, bl.resolver, list, ehlo); err != nil { + return err + } + } + + if list.MAILFROM && mailFrom != "" { + _, domain, err := address.Split(mailFrom) + if err != nil || domain == "" { + // Probably or <>, not much we can check. + return nil + } + + // If EHLO == domain (usually the case for small/private email servers) + // then don't do a second lookup for the same domain. + if list.EHLO && dns.Equal(domain, ehlo) { + return nil + } + + if err := checkDomain(ctx, bl.resolver, list, domain); err != nil { + return err + } + } + + return nil +} + +func (bl *DNSBL) checkLists(ctx context.Context, ip net.IP, ehlo, mailFrom string) module.CheckResult { + var ( + eg = errgroup.Group{} + + // Protects variables below. + lck sync.Mutex + score int + listedOn []string + reasons []string + ) + + for _, list := range bl.bls { + eg.Go(func() error { + err := bl.checkList(ctx, list, ip, ehlo, mailFrom) + if err != nil { + listErr, listed := err.(ListedErr) + if !listed { + return err + } + + lck.Lock() + defer lck.Unlock() + listedOn = append(listedOn, listErr.List) + reasons = append(reasons, listErr.Reason) + score += list.ScoreAdj + } + return nil + }) + } + + err := eg.Wait() + if err != nil { + // Lookup error for BL, hard-fail. + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: exterrors.SMTPCode(err, 451, 554), + EnhancedCode: exterrors.SMTPEnchCode(err, exterrors.EnhancedCode{0, 7, 0}), + Message: "DNS error during policy check", + Err: err, + CheckName: "dnsbl", + }, + } + } + + if score >= bl.rejectThres { + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Client identity is listed in the used DNSBL", + Err: err, + CheckName: "dnsbl", + }, + } + } + if score >= bl.quarantineThres { + return module.CheckResult{ + Quarantine: true, + Reason: &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Client identity is listed in the used DNSBL", + Err: err, + CheckName: "dnsbl", + }, + } + } + + return module.CheckResult{} +} + +// CheckConnection implements module.EarlyCheck. +func (bl *DNSBL) CheckConnection(ctx context.Context, state *module.ConnState) error { + defer trace.StartRegion(ctx, "dnsbl/CheckConnection (Early)").End() + + ip, ok := state.RemoteAddr.(*net.TCPAddr) + if !ok { + bl.log.Msg("non-TCP/IP source", + "src_addr", state.RemoteAddr, + "src_host", state.Hostname) + return nil + } + + result := bl.checkLists(ctx, ip.IP, state.Hostname, "") + if result.Reject && bl.checkEarly { + return result.Reason + } + + state.ModData.Set(bl, true, result) + + return nil +} + +type state struct { + bl *DNSBL + msgMeta *module.MsgMetadata + log log.Logger +} + +func (bl *DNSBL) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &state{ + bl: bl, + msgMeta: msgMeta, + log: target.DeliveryLogger(bl.log, msgMeta), + }, nil +} + +func (s *state) CheckConnection(ctx context.Context) module.CheckResult { + defer trace.StartRegion(ctx, "dnsbl/CheckConnection").End() + + if s.msgMeta.Conn == nil { + s.log.Msg("locally generated message, ignoring") + return module.CheckResult{} + } + + result := s.msgMeta.Conn.ModData.Get(s.bl, true) + if result != nil { + return result.(module.CheckResult) + } + + return module.CheckResult{} +} + +func (*state) CheckSender(context.Context, string) module.CheckResult { + return module.CheckResult{} +} + +func (*state) CheckRcpt(context.Context, string) module.CheckResult { + return module.CheckResult{} +} + +func (*state) CheckBody(context.Context, textproto.Header, buffer.Buffer) module.CheckResult { + return module.CheckResult{} +} + +func (*state) Close() error { + return nil +} + +func init() { + module.Register("check.dnsbl", NewDNSBL) +} diff --git a/internal/check/dnsbl/dnsbl_test.go b/internal/check/dnsbl/dnsbl_test.go new file mode 100644 index 0000000..1845aeb --- /dev/null +++ b/internal/check/dnsbl/dnsbl_test.go @@ -0,0 +1,213 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dnsbl + +import ( + "context" + "errors" + "net" + "testing" + + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestCheckList(t *testing.T) { + test := func(zones map[string]mockdns.Zone, cfg List, ip net.IP, ehlo, mailFrom string, expectedErr error) { + mod := &DNSBL{ + resolver: &mockdns.Resolver{Zones: zones}, + log: testutils.Logger(t, "dnsbl"), + } + err := mod.checkList(context.Background(), cfg, ip, ehlo, mailFrom) + if !errors.Is(err, expectedErr) { + t.Errorf("expected err to be '%#v', got '%#v'", expectedErr, err) + } + } + + test(nil, List{Zone: "example.org"}, net.IPv4(1, 2, 3, 4), + "example.com", "foo@example.com", nil) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", ClientIPv4: true}, net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Reason: "127.0.0.1", + }, + ) + test(map[string]mockdns.Zone{ + "mx.example.com.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org"}, net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", nil, + ) + test(map[string]mockdns.Zone{ + "mx.example.com.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", EHLO: true}, net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", ListedErr{ + Identity: "mx.example.com", + List: "example.org", + Reason: "127.0.0.1", + }, + ) + test(map[string]mockdns.Zone{ + "[1.2.3.4].example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", EHLO: true}, net.IPv4(1, 2, 3, 4), + "[1.2.3.4]", "foo@example.com", nil, + ) + test(map[string]mockdns.Zone{ + "[IPv6:beef::1].example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", EHLO: true}, net.IPv4(1, 2, 3, 4), + "[IPv6:beef::1]", "foo@example.com", nil, + ) + test(map[string]mockdns.Zone{ + "example.com.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org"}, net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", nil, + ) + test(map[string]mockdns.Zone{ + "postmaster.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", MAILFROM: true}, net.IPv4(1, 2, 3, 4), + "mx.example.com", "postmaster", nil, + ) + test(map[string]mockdns.Zone{ + ".example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", MAILFROM: true}, net.IPv4(1, 2, 3, 4), + "mx.example.com", "", nil, + ) + test(map[string]mockdns.Zone{ + "example.com.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", MAILFROM: true}, net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", ListedErr{ + Identity: "example.com", + List: "example.org", + Reason: "127.0.0.1", + }, + ) +} + +func TestCheckLists(t *testing.T) { + test := func(zones map[string]mockdns.Zone, bls []List, ip net.IP, ehlo, mailFrom string, reject, quarantine bool) { + mod := &DNSBL{ + bls: bls, + resolver: &mockdns.Resolver{Zones: zones}, + log: testutils.Logger(t, "dnsbl"), + quarantineThres: 1, + rejectThres: 2, + } + result := mod.checkLists(context.Background(), ip, ehlo, mailFrom) + + if result.Reject && !reject { + t.Errorf("Expected message to not be rejected") + } + if !result.Reject && reject { + t.Errorf("Expected message to be rejected") + } + if result.Quarantine && !quarantine { + t.Errorf("Expected message to not be quarantined") + } + if !result.Quarantine && quarantine { + t.Errorf("Expected message to be quarantined") + } + } + + // Score 2 >= 2, reject + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, []List{ + { + Zone: "example.org", + ClientIPv4: true, + ScoreAdj: 2, + }, + }, net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", true, false, + ) + + // Score 1 >= 1, quarantine + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, []List{ + { + Zone: "example.org", + ClientIPv4: true, + ScoreAdj: 1, + }, + }, net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", false, true, + ) + + // Score 0, no action + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + }, + "4.3.2.1.example.net.": { + A: []string{"127.0.0.1"}, + }, + }, + []List{ + {Zone: "example.org", ClientIPv4: true, ScoreAdj: 1}, + {Zone: "example.net", ClientIPv4: true, ScoreAdj: -1}, + }, + net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", + false, false, + ) + + // DNS error, hard-fail (reject) + test(map[string]mockdns.Zone{ + "4.3.2.2.example.org.": { + Err: &net.DNSError{ + Err: "i/o timeout", + IsTimeout: true, + IsTemporary: true, + }, + }, + }, + []List{ + {Zone: "example.org", ClientIPv4: true, ScoreAdj: 1}, + {Zone: "example.net", ClientIPv4: true, ScoreAdj: 2}, + }, + net.IPv4(2, 2, 3, 4), + "mx.example.com", "foo@example.com", + true, false, + ) +} diff --git a/internal/check/milter/milter.go b/internal/check/milter/milter.go new file mode 100644 index 0000000..37704d4 --- /dev/null +++ b/internal/check/milter/milter.go @@ -0,0 +1,446 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package milter + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-milter" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +const modName = "check.milter" + +type Check struct { + cl *milter.Client + milterUrl string + failOpen bool + instName string + log log.Logger +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + c := &Check{ + instName: instName, + log: log.Logger{Name: modName, Debug: log.DefaultLogger.Debug}, + } + switch len(inlineArgs) { + case 1: + c.milterUrl = inlineArgs[0] + case 0: + default: + return nil, fmt.Errorf("%s: unexpected amount of arguments, want 1 or 0", modName) + } + return c, nil +} + +func (c *Check) Name() string { + return modName +} + +func (c *Check) InstanceName() string { + return c.instName +} + +func (c *Check) Init(cfg *config.Map) error { + cfg.String("endpoint", false, false, c.milterUrl, &c.milterUrl) + cfg.Bool("fail_open", false, false, &c.failOpen) + if _, err := cfg.Process(); err != nil { + return err + } + + if c.milterUrl == "" { + return fmt.Errorf("%s: milter endpoint is not set", modName) + } + + endp, err := config.ParseEndpoint(c.milterUrl) + if err != nil { + return fmt.Errorf("%s: %v", modName, err) + } + + switch endp.Scheme { + case "tcp", "unix": + default: + return fmt.Errorf("%s: scheme unsupported: %v", modName, endp.Scheme) + } + + c.cl = milter.NewClientWithOptions(endp.Network(), endp.Address(), milter.ClientOptions{ + Dialer: &net.Dialer{ + Timeout: 10 * time.Second, + }, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + ActionMask: milter.OptAddHeader | milter.OptQuarantine, + ProtocolMask: 0, + }) + + return nil +} + +type state struct { + c *Check + session *milter.ClientSession + msgMeta *module.MsgMetadata + skipChecks bool + log log.Logger +} + +func (c *Check) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + session, err := c.cl.Session() + if err != nil { + return nil, err + } + return &state{ + c: c, + session: session, + msgMeta: msgMeta, + log: target.DeliveryLogger(c.log, msgMeta), + }, nil +} + +func (s *state) handleAction(act *milter.Action) module.CheckResult { + switch act.Code { + case milter.ActAccept: + s.skipChecks = true + return module.CheckResult{} + case milter.ActContinue: + return module.CheckResult{} + case milter.ActReplyCode: + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: act.SMTPCode, + EnhancedCode: exterrors.EnhancedCode{5, 7, 1}, + Message: "Message rejected due to local policy", + Reason: "reply code action", + CheckName: "milter", + Misc: map[string]interface{}{ + "milter": s.c.milterUrl, + }, + }, + } + case milter.ActDiscard: + s.log.Msg("silent discard is not supported, rejecting message") + fallthrough + case milter.ActTempFail: + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 450, + EnhancedCode: exterrors.EnhancedCode{4, 7, 1}, + Message: "Message rejected due to local policy", + Reason: "reject action", + CheckName: "milter", + Misc: map[string]interface{}{ + "milter": s.c.milterUrl, + }, + }, + } + case milter.ActReject: + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 1}, + Message: "Message rejected due to local policy", + Reason: "reject action", + CheckName: "milter", + Misc: map[string]interface{}{ + "milter": s.c.milterUrl, + }, + }, + } + default: + s.log.Msg("unknown action code ignored", "code", act.Code, "milter", s.c.milterUrl) + return module.CheckResult{} + } +} + +// apply applies the modification actions returned by milter to the check results object. +func (s *state) apply(modifyActs []milter.ModifyAction, res module.CheckResult) module.CheckResult { + out := res + for _, act := range modifyActs { + switch act.Code { + case milter.ActAddRcpt, milter.ActDelRcpt: + s.log.Msg("envelope changes are not supported", "rcpt", act.Rcpt, "code", act.Code, "milter", s.c.milterUrl) + case milter.ActChangeFrom: + s.log.Msg("envelope changes are not supported", "from", act.From, "code", act.Code, "milter", s.c.milterUrl) + case milter.ActChangeHeader: + s.log.Msg("header field changes are not supported", "field", act.HeaderName, "milter", s.c.milterUrl) + case milter.ActInsertHeader: + if act.HeaderIndex != 1 { + s.log.Msg("header inserting not on top is not supported, prepending instead", "field", act.HeaderName, "milter", s.c.milterUrl) + } + fallthrough + case milter.ActAddHeader: + // Header field might be arbitarly folded by the caller and we want + // to preserve that exact format in case it is important (DKIM + // signature is added by milter). + field := make([]byte, 0, len(act.HeaderName)+2+len(act.HeaderValue)+2) + field = append(field, act.HeaderName...) + field = append(field, ':', ' ') + field = append(field, act.HeaderValue...) + field = append(field, '\r', '\n') + out.Header.AddRaw(field) + case milter.ActQuarantine: + out.Quarantine = true + out.Reason = exterrors.WithFields(errors.New("milter quarantine action"), map[string]interface{}{ + "check": "milter", + "milter": s.c.milterUrl, + "reason": act.Reason, + }) + } + } + return out +} + +func (s *state) CheckConnection(ctx context.Context) module.CheckResult { + if s.msgMeta.Conn == nil { + // Submit some dummy values as the message is likely generated locally. + + act, err := s.session.Conn("localhost", milter.FamilyInet, 25, "127.0.0.1") + if err != nil { + return s.ioError(err) + } + if act.Code != milter.ActContinue { + return s.handleAction(act) + } + + act, err = s.session.Helo("localhost") + if err != nil { + return s.ioError(err) + } + return s.handleAction(act) + } + + if !s.session.ProtocolOption(milter.OptNoConnect) { + if err := s.session.Macros(milter.CodeConn, + "daemon_name", "maddy", + "if_name", "unknown", + "if_addr", "0.0.0.0", + // TODO: $j + // TODO: $_ + ); err != nil { + return s.ioError(err) + } + + var ( + protoFamily milter.ProtoFamily + port uint16 + addr string + ) + switch rAddr := s.msgMeta.Conn.RemoteAddr.(type) { + case *net.TCPAddr: + port = uint16(rAddr.Port) + if v4 := rAddr.IP.To4(); v4 != nil { + // Make sure to not accidentally send IPv6-mapped IPv4 address. + protoFamily = milter.FamilyInet + addr = v4.String() + } else { + protoFamily = milter.FamilyInet6 + addr = rAddr.IP.String() + } + case *net.UnixAddr: + protoFamily = milter.FamilyUnix + addr = rAddr.Name + default: + protoFamily = milter.FamilyUnknown + } + + act, err := s.session.Conn(s.msgMeta.Conn.Hostname, protoFamily, port, addr) + if err != nil { + return s.ioError(err) + } + if act.Code != milter.ActContinue { + return s.handleAction(act) + } + } + + if !s.session.ProtocolOption(milter.OptNoHelo) { + if s.msgMeta.Conn.TLS.HandshakeComplete { + fields := make([]string, 0, 4*2) + tlsState := s.msgMeta.Conn.TLS + + switch tlsState.Version { + case tls.VersionTLS10: + fields = append(fields, "tls_version", "TLSv1") + case tls.VersionTLS11: + fields = append(fields, "tls_version", "TLSv1.1") + case tls.VersionTLS12: + fields = append(fields, "tls_version", "TLSv1.2") + case tls.VersionTLS13: + fields = append(fields, "tls_version", "TLSv1.3") + } + fields = append(fields, "cipher", tls.CipherSuiteName(tlsState.CipherSuite)) + + if len(tlsState.PeerCertificates) != 0 { + fields = append(fields, "cert_subject", + tlsState.PeerCertificates[len(tlsState.PeerCertificates)-1].Subject.String()) + fields = append(fields, "cert_issuer", + tlsState.PeerCertificates[len(tlsState.PeerCertificates)-1].Issuer.String()) + } + + if err := s.session.Macros(milter.CodeHelo, fields...); err != nil { + return s.ioError(err) + } + } + act, err := s.session.Helo(s.msgMeta.Conn.Hostname) + if err != nil { + return s.ioError(err) + } + return s.handleAction(act) + } + + return module.CheckResult{} +} + +func (s *state) ioError(err error) module.CheckResult { + if s.c.failOpen { + s.skipChecks = true // silently permit processing to continue + s.c.log.Error("I/O error", err) + return module.CheckResult{} + } + + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 1}, + Message: "I/O error during policy check", + Err: err, + CheckName: "milter", + Misc: map[string]interface{}{ + "milter": s.c.milterUrl, + }, + }, + } +} + +func (s *state) CheckSender(ctx context.Context, mailFrom string) module.CheckResult { + if s.skipChecks || s.session.ProtocolOption(milter.OptNoMailFrom) { + return module.CheckResult{} + } + + fields := make([]string, 0, 2) + fields = append(fields, "i", s.msgMeta.ID) + // TODO: fields = append(fields, "auth_type", s.msgMeta.???) + if s.msgMeta.Conn.AuthUser != "" { + fields = append(fields, "auth_authen", s.msgMeta.Conn.AuthUser) + } + if err := s.session.Macros(milter.CodeMail, fields...); err != nil { + return s.ioError(err) + } + + esmtpArgs := make([]string, 0, 2) + if s.msgMeta.SMTPOpts.UTF8 { + esmtpArgs = append(esmtpArgs, "SMTPUTF8") + } + + act, err := s.session.Mail(mailFrom, esmtpArgs) + if err != nil { + return s.ioError(err) + } + return s.handleAction(act) +} + +func (s *state) CheckRcpt(ctx context.Context, rcptTo string) module.CheckResult { + if s.skipChecks { + return module.CheckResult{} + } + + act, err := s.session.Rcpt(rcptTo, nil) + if err != nil { + return s.ioError(err) + } + return s.handleAction(act) +} + +func (s *state) CheckBody(ctx context.Context, header textproto.Header, body buffer.Buffer) module.CheckResult { + if s.skipChecks { + return module.CheckResult{} + } + + act, err := s.session.Header(header) + if err != nil { + return s.ioError(err) + } + if act.Code != milter.ActContinue { + return s.handleAction(act) + } + + var modifyAct []milter.ModifyAction + + if !s.session.ProtocolOption(milter.OptNoBody) { + // body.Open can be expensive for on-disk buffering. + r, err := body.Open() + if err != nil { + // Not ioError(err) because fail_open directive is applied only for external I/O. + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 1}, + Message: "Internal error during policy check", + Err: err, + CheckName: "milter", + Misc: map[string]interface{}{ + "milter": s.c.milterUrl, + }, + }, + } + } + + modifyAct, act, err = s.session.BodyReadFrom(r) + if err != nil { + return s.ioError(err) + } + } else { + modifyAct, act, err = s.session.End() + if err != nil { + return s.ioError(err) + } + } + + result := s.handleAction(act) + return s.apply(modifyAct, result) +} + +func (s *state) Close() error { + return s.session.Close() +} + +var ( + _ module.Check = &Check{} + _ module.CheckState = &state{} +) + +func init() { + module.Register(modName, New) +} diff --git a/internal/check/milter/milter_test.go b/internal/check/milter/milter_test.go new file mode 100644 index 0000000..9797851 --- /dev/null +++ b/internal/check/milter/milter_test.go @@ -0,0 +1,61 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package milter + +import ( + "testing" + + "github.com/foxcpp/maddy/framework/config" +) + +func TestAcceptValidEndpoints(t *testing.T) { + for _, endpoint := range []string{ + "tcp://0.0.0.0:10025", + "tcp://[::]:10025", + "tcp:127.0.0.1:10025", + "unix://path", + "unix:path", + "unix:/path", + "unix:///path", + "unix://also/path", + "unix:///also/path", + } { + c := &Check{milterUrl: endpoint} + + err := c.Init(&config.Map{}) + if err != nil { + t.Errorf("Unexpected failure for %s: %v", endpoint, err) + return + } + } +} + +func TestRejectInvalidEndpoints(t *testing.T) { + for _, endpoint := range []string{ + "tls://0.0.0.0:10025", + "tls:0.0.0.0:10025", + } { + c := &Check{milterUrl: endpoint} + err := c.Init(&config.Map{}) + if err == nil { + t.Errorf("Accepted invalid endpoint: %s", endpoint) + return + } + } +} diff --git a/internal/check/requiretls/requiretls.go b/internal/check/requiretls/requiretls.go new file mode 100644 index 0000000..bdd2f26 --- /dev/null +++ b/internal/check/requiretls/requiretls.go @@ -0,0 +1,45 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package requiretls + +import ( + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/check" +) + +func requireTLS(ctx check.StatelessCheckContext) module.CheckResult { + if ctx.MsgMeta.Conn != nil && ctx.MsgMeta.Conn.TLS.HandshakeComplete { + return module.CheckResult{} + } + + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 1}, + Message: "TLS conversation required", + CheckName: "require_tls", + }, + } +} + +func init() { + check.RegisterStatelessCheck("require_tls", modconfig.FailAction{Reject: true}, requireTLS, nil, nil, nil) +} diff --git a/internal/check/rspamd/rspamd.go b/internal/check/rspamd/rspamd.go new file mode 100644 index 0000000..e6afad6 --- /dev/null +++ b/internal/check/rspamd/rspamd.go @@ -0,0 +1,367 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package rspamd + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strconv" + "strings" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + tls2 "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +const modName = "check.rspamd" + +type Check struct { + instName string + log log.Logger + + apiPath string + flags string + settingsID string + tag string + mtaName string + + ioErrAction modconfig.FailAction + errorRespAction modconfig.FailAction + addHdrAction modconfig.FailAction + rewriteSubjAction modconfig.FailAction + + client *http.Client +} + +func New(modName, instName string, _, inlineArgs []string) (module.Module, error) { + c := &Check{ + instName: instName, + client: http.DefaultClient, + log: log.Logger{Name: modName, Debug: log.DefaultLogger.Debug}, + } + + switch len(inlineArgs) { + case 1: + c.apiPath = inlineArgs[0] + case 0: + c.apiPath = "http://127.0.0.1:11333" + default: + return nil, fmt.Errorf("%s: unexpected amount of inline arguments", modName) + } + + return c, nil +} + +func (c *Check) Name() string { + return modName +} + +func (c *Check) InstanceName() string { + return c.instName +} + +func (c *Check) Init(cfg *config.Map) error { + var ( + tlsConfig tls.Config + flags []string + ) + + cfg.Custom("tls_client", true, false, func() (interface{}, error) { + return tls.Config{}, nil + }, tls2.TLSClientBlock, &tlsConfig) + cfg.String("api_path", false, false, c.apiPath, &c.apiPath) + cfg.String("settings_id", false, false, "", &c.settingsID) + cfg.String("tag", false, false, "maddy", &c.tag) + cfg.String("hostname", true, false, "", &c.mtaName) + cfg.Custom("io_error_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.ioErrAction) + cfg.Custom("error_resp_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.errorRespAction) + cfg.Custom("add_header_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{Quarantine: true}, nil + }, modconfig.FailActionDirective, &c.addHdrAction) + cfg.Custom("rewrite_subj_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{Quarantine: true}, nil + }, modconfig.FailActionDirective, &c.rewriteSubjAction) + cfg.StringList("flags", false, false, []string{"pass_all"}, &flags) + if _, err := cfg.Process(); err != nil { + return err + } + + c.client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tlsConfig, + }, + } + c.flags = strings.Join(flags, ",") + + return nil +} + +type state struct { + c *Check + msgMeta *module.MsgMetadata + log log.Logger + + mailFrom string + rcpt []string +} + +func (c *Check) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &state{ + c: c, + msgMeta: msgMeta, + log: target.DeliveryLogger(c.log, msgMeta), + }, nil +} + +func (s *state) CheckConnection(ctx context.Context) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckSender(ctx context.Context, addr string) module.CheckResult { + s.mailFrom = addr + return module.CheckResult{} +} + +func (s *state) CheckRcpt(ctx context.Context, addr string) module.CheckResult { + s.rcpt = append(s.rcpt, addr) + return module.CheckResult{} +} + +func addConnHeaders(r *http.Request, meta *module.MsgMetadata, mailFrom string, rcpts []string) { + r.Header.Add("From", mailFrom) + for _, rcpt := range rcpts { + r.Header.Add("Rcpt", rcpt) + } + + r.Header.Add("Queue-ID", meta.ID) + + conn := meta.Conn + if conn != nil { + if meta.Conn.AuthUser != "" { + r.Header.Add("User", meta.Conn.AuthUser) + } + + if tcpAddr, ok := conn.RemoteAddr.(*net.TCPAddr); ok { + r.Header.Add("IP", tcpAddr.IP.String()) + } + r.Header.Add("Helo", conn.Hostname) + name, err := conn.RDNSName.Get() + if err == nil && name != nil { + r.Header.Add("Hostname", name.(string)) + } + + if conn.TLS.HandshakeComplete { + r.Header.Add("TLS-Cipher", tls.CipherSuiteName(conn.TLS.CipherSuite)) + switch conn.TLS.Version { + case tls.VersionTLS13: + r.Header.Add("TLS-Version", "1.3") + case tls.VersionTLS12: + r.Header.Add("TLS-Version", "1.2") + case tls.VersionTLS11: + r.Header.Add("TLS-Version", "1.1") + case tls.VersionTLS10: + r.Header.Add("TLS-Version", "1.0") + } + } + } +} + +func (s *state) CheckBody(ctx context.Context, hdr textproto.Header, body buffer.Buffer) module.CheckResult { + bodyR, err := body.Open() + if err != nil { + return module.CheckResult{ + Reject: true, + Reason: exterrors.WithFields(err, map[string]interface{}{"check": modName}), + } + } + + var buf bytes.Buffer + if err := textproto.WriteHeader(&buf, hdr); err != nil { + return module.CheckResult{ + Reject: true, + Reason: exterrors.WithFields(err, map[string]interface{}{"check": modName}), + } + } + + r, err := http.NewRequest("POST", s.c.apiPath+"/checkv2", io.MultiReader(&buf, bodyR)) + if err != nil { + return module.CheckResult{ + Reject: true, + Reason: exterrors.WithFields(err, map[string]interface{}{"check": modName}), + } + } + + r.Header.Add("Pass", "all") // TODO: does that need to be configurable? + // TODO: include version (needs maddy.Version moved somewhere to break circular dependency) + r.Header.Add("User-Agent", "maddy") + if s.c.tag != "" { + r.Header.Add("MTA-Tag", s.c.tag) + } + if s.c.settingsID != "" { + r.Header.Add("Settings-ID", s.c.settingsID) + } + if s.c.mtaName != "" { + r.Header.Add("MTA-Name", s.c.mtaName) + } + + addConnHeaders(r, s.msgMeta, s.mailFrom, s.rcpt) + r.Header.Add("Content-Length", strconv.Itoa(body.Len())) + + resp, err := s.c.client.Do(r) + if err != nil { + return s.c.ioErrAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Internal error during policy check", + CheckName: modName, + Err: err, + }, + }) + } + if resp.StatusCode/100 != 2 { + return s.c.errorRespAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Internal error during policy check", + CheckName: modName, + Err: fmt.Errorf("HTTP %d", resp.StatusCode), + }, + }) + } + defer resp.Body.Close() + + var respData response + if err := json.NewDecoder(resp.Body).Decode(&respData); err != nil { + return s.c.ioErrAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 9, 0}, + Message: "Internal error during policy check", + CheckName: modName, + Err: err, + }, + }) + } + + switch respData.Action { + case "no action": + return module.CheckResult{} + case "greylist": + // uuh... TODO: Implement greylisting? + hdrAdd := textproto.Header{} + hdrAdd.Add("X-Spam-Score", strconv.FormatFloat(respData.Score, 'f', 2, 64)) + return module.CheckResult{ + Header: hdrAdd, + } + case "add header": + hdrAdd := textproto.Header{} + hdrAdd.Add("X-Spam-Flag", "Yes") + hdrAdd.Add("X-Spam-Score", strconv.FormatFloat(respData.Score, 'f', 2, 64)) + return s.c.addHdrAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 450, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Message rejected due to local policy", + CheckName: modName, + Misc: map[string]interface{}{"action": "add header"}, + }, + Header: hdrAdd, + }) + case "rewrite subject": + hdrAdd := textproto.Header{} + hdrAdd.Add("X-Spam-Flag", "Yes") + hdrAdd.Add("X-Spam-Score", strconv.FormatFloat(respData.Score, 'f', 2, 64)) + return s.c.rewriteSubjAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 450, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Message rejected due to local policy", + CheckName: modName, + Misc: map[string]interface{}{"action": "rewrite subject"}, + }, + Header: hdrAdd, + }) + case "soft reject": + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 450, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Message rejected due to local policy", + CheckName: modName, + Misc: map[string]interface{}{"action": "soft reject"}, + }, + } + case "reject": + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Message rejected due to local policy", + CheckName: modName, + Misc: map[string]interface{}{"action": "reject"}, + }, + } + } + + s.log.Msg("unhandled action", "action", respData.Action) + + return module.CheckResult{} +} + +type response struct { + Score float64 `json:"score"` + Action string `json:"action"` + Subject string `json:"subject"` + Symbols map[string]struct { + Name string `json:"name"` + Score float64 `json:"score"` + } +} + +func (s *state) Close() error { + return nil +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/check/skeleton.go b/internal/check/skeleton.go new file mode 100644 index 0000000..f374359 --- /dev/null +++ b/internal/check/skeleton.go @@ -0,0 +1,101 @@ +//go:build ignore +// +build ignore + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +/* +This is example of a minimal stateful check module implementation. +See HACKING.md in the repo root for implementation recommendations. +*/ + +package directory_name_here + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +const modName = "check_things" + +type Check struct { + instName string + log log.Logger +} + +func New(modName, instName string, aliases, inlineArgs []string) (module.Module, error) { + return &Check{ + instName: instName, + }, nil +} + +func (c *Check) Name() string { + return modName +} + +func (c *Check) InstanceName() string { + return c.instName +} + +func (c *Check) Init(cfg *config.Map) error { + return nil +} + +type state struct { + c *Check + msgMeta *module.MsgMetadata + log log.Logger +} + +func (c *Check) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &state{ + c: c, + msgMeta: msgMeta, + log: target.DeliveryLogger(c.log, msgMeta), + }, nil +} + +func (s *state) CheckConnection(ctx context.Context) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckSender(ctx context.Context, addr string) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckRcpt(ctx context.Context, addr string) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckBody(ctx context.Context, hdr textproto.Header, body buffer.Buffer) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) Close() error { + return nil +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/check/spf/spf.go b/internal/check/spf/spf.go new file mode 100644 index 0000000..c94dd91 --- /dev/null +++ b/internal/check/spf/spf.go @@ -0,0 +1,420 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package spf + +import ( + "context" + "errors" + "fmt" + "net" + "runtime/debug" + "runtime/trace" + + "blitiri.com.ar/go/spf" + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/emersion/go-msgauth/dmarc" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + maddydmarc "github.com/foxcpp/maddy/internal/dmarc" + "github.com/foxcpp/maddy/internal/target" + "golang.org/x/net/idna" +) + +const modName = "check.spf" + +type Check struct { + instName string + enforceEarly bool + + noneAction modconfig.FailAction + neutralAction modconfig.FailAction + failAction modconfig.FailAction + softfailAction modconfig.FailAction + permerrAction modconfig.FailAction + temperrAction modconfig.FailAction + + log log.Logger + resolver dns.Resolver +} + +func New(_, instName string, _, _ []string) (module.Module, error) { + return &Check{ + instName: instName, + log: log.Logger{Name: modName}, + resolver: dns.DefaultResolver(), + }, nil +} + +func (c *Check) Name() string { + return modName +} + +func (c *Check) InstanceName() string { + return c.instName +} + +func (c *Check) Init(cfg *config.Map) error { + cfg.Bool("debug", true, false, &c.log.Debug) + cfg.Bool("enforce_early", true, false, &c.enforceEarly) + cfg.Custom("none_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.noneAction) + cfg.Custom("neutral_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.neutralAction) + cfg.Custom("fail_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{Quarantine: true}, nil + }, modconfig.FailActionDirective, &c.failAction) + cfg.Custom("softfail_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.softfailAction) + cfg.Custom("permerr_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.permerrAction) + cfg.Custom("temperr_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.temperrAction) + _, err := cfg.Process() + if err != nil { + return err + } + + return nil +} + +type spfRes struct { + res spf.Result + err error +} + +type state struct { + c *Check + msgMeta *module.MsgMetadata + spfFetch chan spfRes + log log.Logger + + skip bool +} + +func (c *Check) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &state{ + c: c, + msgMeta: msgMeta, + spfFetch: make(chan spfRes, 1), + log: target.DeliveryLogger(c.log, msgMeta), + }, nil +} + +func (s *state) spfResult(res spf.Result, err error) module.CheckResult { + _, fromDomain, _ := address.Split(s.msgMeta.OriginalFrom) + spfAuth := &authres.SPFResult{ + Value: authres.ResultNone, + Helo: s.msgMeta.Conn.Hostname, + From: fromDomain, + } + + if err != nil { + spfAuth.Reason = err.Error() + } else if res == spf.None { + spfAuth.Reason = "no policy" + } + + switch res { + case spf.None: + spfAuth.Value = authres.ResultNone + return s.c.noneAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 23}, + Message: "No SPF policy", + CheckName: modName, + Err: err, + }, + AuthResult: []authres.Result{spfAuth}, + }) + case spf.Neutral: + spfAuth.Value = authres.ResultNeutral + return s.c.neutralAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 23}, + Message: "Neutral SPF result is not permitted", + CheckName: modName, + Err: err, + }, + AuthResult: []authres.Result{spfAuth}, + }) + case spf.Pass: + spfAuth.Value = authres.ResultPass + return module.CheckResult{AuthResult: []authres.Result{spfAuth}} + case spf.Fail: + spfAuth.Value = authres.ResultFail + return s.c.failAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 23}, + Message: "SPF authentication failed", + CheckName: modName, + Err: err, + }, + AuthResult: []authres.Result{spfAuth}, + }) + case spf.SoftFail: + spfAuth.Value = authres.ResultSoftFail + return s.c.softfailAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 23}, + Message: "SPF authentication soft-failed", + CheckName: modName, + Err: err, + }, + AuthResult: []authres.Result{spfAuth}, + }) + case spf.TempError: + spfAuth.Value = authres.ResultTempError + return s.c.temperrAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 23}, + Message: "SPF authentication failed with a temporary error", + CheckName: modName, + Err: err, + }, + AuthResult: []authres.Result{spfAuth}, + }) + case spf.PermError: + spfAuth.Value = authres.ResultPermError + return s.c.permerrAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 23}, + Message: "SPF authentication failed with a permanent error", + CheckName: modName, + Err: err, + }, + AuthResult: []authres.Result{spfAuth}, + }) + } + + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{4, 7, 23}, + Message: fmt.Sprintf("Unknown SPF status: %s", res), + CheckName: modName, + Err: err, + }, + AuthResult: []authres.Result{spfAuth}, + } +} + +func (s *state) relyOnDMARC(ctx context.Context, hdr textproto.Header) bool { + fromDomain, err := maddydmarc.ExtractFromDomain(hdr) + if err != nil { + s.log.Error("DMARC domains extract", err) + return false + } + + policyDomain, record, err := maddydmarc.FetchRecord(ctx, s.c.resolver, fromDomain) + if err != nil { + s.log.Error("DMARC fetch", err, "from_domain", fromDomain) + return false + } + if record == nil { + return false + } + + policy := record.Policy + // We check for subdomain using non-equality since fromDomain is either the + // subdomain of policyDomain or policyDomain itself (due to the way + // FetchRecord handles it). + if !dns.Equal(policyDomain, fromDomain) && record.SubdomainPolicy != "" { + policy = record.SubdomainPolicy + } + + return policy != dmarc.PolicyNone +} + +func prepareMailFrom(from string) (string, error) { + // INTERNATIONALIZATION: RFC 8616, Section 4 + // Hostname is already in A-labels per SMTPUTF8 requirement. + // MAIL FROM domain should be converted to A-labels before doing + // anything. + fromMbox, fromDomain, err := address.Split(from) + if err != nil { + return "", &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 1, 7}, + Message: "Malformed address", + CheckName: "spf", + } + } + fromDomain, err = idna.ToASCII(fromDomain) + if err != nil { + return "", &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 1, 7}, + Message: "Malformed address", + CheckName: "spf", + } + } + + // %{s} and %{l} do not match anything if it is non-ASCII. + // Since spf lib does not seem to care, strip it. + if !address.IsASCII(fromMbox) { + fromMbox = "" + } + + return fromMbox + "@" + dns.FQDN(fromDomain), nil +} + +func (s *state) CheckConnection(ctx context.Context) module.CheckResult { + defer trace.StartRegion(ctx, "check.spf/CheckConnection").End() + + if s.msgMeta.Conn == nil { + s.skip = true + s.log.Println("locally generated message, skipping") + return module.CheckResult{} + } + + ip, ok := s.msgMeta.Conn.RemoteAddr.(*net.TCPAddr) + if !ok { + s.skip = true + s.log.Println("non-IP SrcAddr") + return module.CheckResult{} + } + + mailFromOriginal := s.msgMeta.OriginalFrom + if mailFromOriginal == "" { + // RFC 7208 Section 2.4. + // >When the reverse-path is null, this document + // >defines the "MAIL FROM" identity to be the mailbox composed of the + // >local-part "postmaster" and the "HELO" identity (which might or might + // >not have been checked separately before). + mailFromOriginal = "postmaster@" + s.msgMeta.Conn.Hostname + } + + mailFrom, err := prepareMailFrom(mailFromOriginal) + if err != nil { + s.skip = true + return module.CheckResult{ + Reason: err, + Reject: true, + } + } + + if s.c.enforceEarly { + res, err := spf.CheckHostWithSender(ip.IP, + dns.FQDN(s.msgMeta.Conn.Hostname), mailFrom, + spf.WithContext(ctx), spf.WithResolver(s.c.resolver)) + s.log.Debugf("result: %s (%v)", res, err) + return s.spfResult(res, err) + } + + // We start evaluation in parallel to other message processing, + // once we get the body, we fetch DMARC policy and see if it exists + // and not p=none. In that case, we rely on DMARC alignment to define result. + // Otherwise, we take action based on SPF only. + + go func() { + defer func() { + if err := recover(); err != nil { + stack := debug.Stack() + log.Printf("panic during spf.CheckHostWithSender: %v\n%s", err, stack) + close(s.spfFetch) + } + }() + + defer trace.StartRegion(ctx, "check.spf/CheckConnection (Async)").End() + + res, err := spf.CheckHostWithSender(ip.IP, dns.FQDN(s.msgMeta.Conn.Hostname), mailFrom, + spf.WithContext(ctx), spf.WithResolver(s.c.resolver)) + s.log.Debugf("result: %s (%v)", res, err) + s.spfFetch <- spfRes{res, err} + }() + + return module.CheckResult{} +} + +func (s *state) CheckSender(ctx context.Context, mailFrom string) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckRcpt(ctx context.Context, rcptTo string) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckBody(ctx context.Context, header textproto.Header, body buffer.Buffer) module.CheckResult { + if s.c.enforceEarly || s.skip { + // Already applied in CheckConnection. + return module.CheckResult{} + } + + defer trace.StartRegion(ctx, "check.spf/CheckBody").End() + + res, ok := <-s.spfFetch + if !ok { + return module.CheckResult{ + Reject: true, + Reason: exterrors.WithTemporary( + exterrors.WithFields(errors.New("panic recovered"), map[string]interface{}{ + "check": "spf", + "smtp_msg": "Internal error during policy check", + }), + true, + ), + } + } + if s.relyOnDMARC(ctx, header) { + if res.res != spf.Pass { + s.log.Msg("deferring action due to a DMARC policy", "result", res.res, "err", res.err) + } else { + s.log.DebugMsg("deferring action due to a DMARC policy", "result", res.res, "err", res.err) + } + + checkRes := s.spfResult(res.res, res.err) + checkRes.Quarantine = false + checkRes.Reject = false + return checkRes + } + + return s.spfResult(res.res, res.err) +} + +func (s *state) Close() error { + return nil +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/check/stateless_check.go b/internal/check/stateless_check.go new file mode 100644 index 0000000..4c5d2d5 --- /dev/null +++ b/internal/check/stateless_check.go @@ -0,0 +1,202 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package check + +import ( + "context" + "fmt" + "runtime/trace" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +type ( + StatelessCheckContext struct { + // Embedded context.Context value, used for tracing, cancellation and + // timeouts. + context.Context + + // Resolver that should be used by the check for DNS queries. + Resolver dns.Resolver + + MsgMeta *module.MsgMetadata + + // Logger that should be used by the check for logging, note that it is + // already wrapped to append Msg ID to all messages so check code + // should not do the same. + Logger log.Logger + } + FuncConnCheck func(checkContext StatelessCheckContext) module.CheckResult + FuncSenderCheck func(checkContext StatelessCheckContext, mailFrom string) module.CheckResult + FuncRcptCheck func(checkContext StatelessCheckContext, rcptTo string) module.CheckResult + FuncBodyCheck func(checkContext StatelessCheckContext, header textproto.Header, body buffer.Buffer) module.CheckResult +) + +type statelessCheck struct { + modName string + instName string + resolver dns.Resolver + logger log.Logger + + // One used by Init if config option is not passed by a user. + defaultFailAction modconfig.FailAction + // The actual fail action that should be applied. + failAction modconfig.FailAction + + connCheck FuncConnCheck + senderCheck FuncSenderCheck + rcptCheck FuncRcptCheck + bodyCheck FuncBodyCheck +} + +type statelessCheckState struct { + c *statelessCheck + msgMeta *module.MsgMetadata +} + +func (s *statelessCheckState) String() string { + return s.c.modName + ":" + s.c.instName +} + +func (s *statelessCheckState) CheckConnection(ctx context.Context) module.CheckResult { + if s.c.connCheck == nil { + return module.CheckResult{} + } + defer trace.StartRegion(ctx, s.c.modName+"/CheckConnection").End() + + originalRes := s.c.connCheck(StatelessCheckContext{ + Context: ctx, + Resolver: s.c.resolver, + MsgMeta: s.msgMeta, + Logger: target.DeliveryLogger(s.c.logger, s.msgMeta), + }) + return s.c.failAction.Apply(originalRes) +} + +func (s *statelessCheckState) CheckSender(ctx context.Context, mailFrom string) module.CheckResult { + if s.c.senderCheck == nil { + return module.CheckResult{} + } + defer trace.StartRegion(ctx, s.c.modName+"/CheckSender").End() + + originalRes := s.c.senderCheck(StatelessCheckContext{ + Context: ctx, + Resolver: s.c.resolver, + MsgMeta: s.msgMeta, + Logger: target.DeliveryLogger(s.c.logger, s.msgMeta), + }, mailFrom) + return s.c.failAction.Apply(originalRes) +} + +func (s *statelessCheckState) CheckRcpt(ctx context.Context, rcptTo string) module.CheckResult { + if s.c.rcptCheck == nil { + return module.CheckResult{} + } + defer trace.StartRegion(ctx, s.c.modName+"/CheckRcpt").End() + + originalRes := s.c.rcptCheck(StatelessCheckContext{ + Context: ctx, + Resolver: s.c.resolver, + MsgMeta: s.msgMeta, + Logger: target.DeliveryLogger(s.c.logger, s.msgMeta), + }, rcptTo) + return s.c.failAction.Apply(originalRes) +} + +func (s *statelessCheckState) CheckBody(ctx context.Context, header textproto.Header, body buffer.Buffer) module.CheckResult { + if s.c.bodyCheck == nil { + return module.CheckResult{} + } + defer trace.StartRegion(ctx, s.c.modName+"/CheckBody").End() + + originalRes := s.c.bodyCheck(StatelessCheckContext{ + Context: ctx, + Resolver: s.c.resolver, + MsgMeta: s.msgMeta, + Logger: target.DeliveryLogger(s.c.logger, s.msgMeta), + }, header, body) + return s.c.failAction.Apply(originalRes) +} + +func (s *statelessCheckState) Close() error { + return nil +} + +func (c *statelessCheck) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &statelessCheckState{ + c: c, + msgMeta: msgMeta, + }, nil +} + +func (c *statelessCheck) Init(cfg *config.Map) error { + cfg.Bool("debug", true, false, &c.logger.Debug) + cfg.Custom("fail_action", false, false, + func() (interface{}, error) { + return c.defaultFailAction, nil + }, modconfig.FailActionDirective, &c.failAction) + _, err := cfg.Process() + return err +} + +func (c *statelessCheck) Name() string { + return c.modName +} + +func (c *statelessCheck) InstanceName() string { + return c.instName +} + +// RegisterStatelessCheck is helper function to create stateless message check modules +// that run one simple check during one stage. +// +// It creates the module and its instance with the specified name that implement module.Check interface +// and runs passed functions when corresponding module.CheckState methods are called. +// +// Note about CheckResult that is returned by the functions: +// StatelessCheck supports different action types based on the user configuration, but the particular check +// code doesn't need to know about it. It should assume that it is always "Reject" and hence it should +// populate Reason field of the result object with the relevant error description. +func RegisterStatelessCheck(name string, defaultFailAction modconfig.FailAction, connCheck FuncConnCheck, senderCheck FuncSenderCheck, rcptCheck FuncRcptCheck, bodyCheck FuncBodyCheck) { + module.Register(name, func(modName, instName string, aliases, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, fmt.Errorf("%s: inline arguments are not used", modName) + } + return &statelessCheck{ + modName: modName, + instName: instName, + resolver: dns.DefaultResolver(), + logger: log.Logger{Name: modName}, + + defaultFailAction: defaultFailAction, + + connCheck: connCheck, + senderCheck: senderCheck, + rcptCheck: rcptCheck, + bodyCheck: bodyCheck, + }, nil + }) +} diff --git a/internal/cli/app.go b/internal/cli/app.go new file mode 100644 index 0000000..5910896 --- /dev/null +++ b/internal/cli/app.go @@ -0,0 +1,113 @@ +package maddycli + +import ( + "fmt" + "os" + "strings" + + "github.com/foxcpp/maddy/framework/log" + "github.com/urfave/cli/v2" +) + +var app *cli.App + +func init() { + app = cli.NewApp() + app.Usage = "composable all-in-one mail server" + app.Description = `Maddy is Mail Transfer agent (MTA), Mail Delivery Agent (MDA), Mail Submission +Agent (MSA), IMAP server and a set of other essential protocols/schemes +necessary to run secure email server implemented in one executable. + +This executable can be used to start the server ('run') and to manipulate +databases used by it (all other subcommands). +` + app.Authors = []*cli.Author{ + { + Name: "Maddy Mail Server maintainers & contributors", + Email: "~foxcpp/maddy@lists.sr.ht", + }, + } + app.ExitErrHandler = func(c *cli.Context, err error) { + cli.HandleExitCoder(err) + } + app.EnableBashCompletion = true + app.Commands = []*cli.Command{ + { + Name: "generate-man", + Hidden: true, + Action: func(c *cli.Context) error { + man, err := app.ToMan() + if err != nil { + return err + } + fmt.Println(man) + return nil + }, + }, + { + Name: "generate-fish-completion", + Hidden: true, + Action: func(c *cli.Context) error { + cp, err := app.ToFishCompletion() + if err != nil { + return err + } + fmt.Println(cp) + return nil + }, + }, + } +} + +func AddGlobalFlag(f cli.Flag) { + app.Flags = append(app.Flags, f) +} + +func AddSubcommand(cmd *cli.Command) { + app.Commands = append(app.Commands, cmd) + + if cmd.Name == "run" { + // Backward compatibility hack to start the server as just ./maddy + // Needs to be done here so we will register all known flags with + // stdlib before Run is called. + app.Action = func(c *cli.Context) error { + log.Println("WARNING: Starting server not via 'maddy run' is deprecated and will stop working in the next version") + return cmd.Action(c) + } + app.Flags = append(app.Flags, cmd.Flags...) + } +} + +// RunWithoutExit is like Run but returns exit code instead of calling os.Exit +// To be used in maddy.cover. +func RunWithoutExit() int { + code := 0 + + cli.OsExiter = func(c int) { code = c } + defer func() { + cli.OsExiter = os.Exit + }() + + Run() + + return code +} + +func Run() { + mapStdlibFlags(app) + + // Actual entry point is registered in maddy.go. + + // Print help when called via maddyctl executable. To be removed + // once backward compatibility hack for 'maddy run' is removed too. + if strings.Contains(os.Args[0], "maddyctl") && len(os.Args) == 1 { + if err := app.Run([]string{os.Args[0], "help"}); err != nil { + log.DefaultLogger.Error("app.Run failed", err) + } + return + } + + if err := app.Run(os.Args); err != nil { + log.DefaultLogger.Error("app.Run failed", err) + } +} diff --git a/internal/cli/clitools/clitools.go b/internal/cli/clitools/clitools.go new file mode 100644 index 0000000..2da6998 --- /dev/null +++ b/internal/cli/clitools/clitools.go @@ -0,0 +1,118 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package clitools + +import ( + "bufio" + "errors" + "fmt" + "os" +) + +var stdinScanner = bufio.NewScanner(os.Stdin) + +func Confirmation(prompt string, def bool) bool { + selection := "y/N" + if def { + selection = "Y/n" + } + + fmt.Fprintf(os.Stderr, "%s [%s]: ", prompt, selection) + if !stdinScanner.Scan() { + fmt.Fprintln(os.Stderr, stdinScanner.Err()) + return false + } + + switch stdinScanner.Text() { + case "Y", "y": + return true + case "N", "n": + return false + default: + return def + } +} + +func readPass(tty *os.File, output []byte) ([]byte, error) { + cursor := output[0:1] + readen := 0 + for { + n, err := tty.Read(cursor) + if n != 1 { + return nil, errors.New("ReadPassword: invalid read size when not in canonical mode") + } + if err != nil { + return nil, errors.New("ReadPassword: " + err.Error()) + } + if cursor[0] == '\n' { + break + } + // Esc or Ctrl+D or Ctrl+C. + if cursor[0] == '\x1b' || cursor[0] == '\x04' || cursor[0] == '\x03' { + return nil, errors.New("ReadPassword: prompt rejected") + } + if cursor[0] == '\x7F' /* DEL */ { + if readen != 0 { + readen-- + cursor = output[readen : readen+1] + } + continue + } + + if readen == cap(output) { + return nil, errors.New("ReadPassword: too long password") + } + + readen++ + cursor = output[readen : readen+1] + } + + return output[0:readen], nil +} + +func ReadPassword(prompt string) (string, error) { + termios, err := TurnOnRawIO(os.Stdin) + hiddenPass := true + if err != nil { + hiddenPass = false + fmt.Fprintln(os.Stderr, "Failed to disable terminal output:", err) + } + + // There is no meaningful way to handle error here. + //nolint:errcheck + defer TcSetAttr(os.Stdin.Fd(), &termios) + + fmt.Fprintf(os.Stderr, "%s: ", prompt) + + if hiddenPass { + buf := make([]byte, 512) + buf, err = readPass(os.Stdin, buf) + if err != nil { + return "", err + } + fmt.Println() + + return string(buf), nil + } + if !stdinScanner.Scan() { + return "", stdinScanner.Err() + } + + return stdinScanner.Text(), nil +} diff --git a/internal/cli/clitools/termios.go b/internal/cli/clitools/termios.go new file mode 100644 index 0000000..cf817d1 --- /dev/null +++ b/internal/cli/clitools/termios.go @@ -0,0 +1,82 @@ +//go:build linux +// +build linux + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package clitools + +// Copied from github.com/foxcpp/ttyprompt +// Commit 087a574, terminal/termios.go + +import ( + "errors" + "os" + "syscall" + "unsafe" +) + +type Termios struct { + Iflag uint32 + Oflag uint32 + Cflag uint32 + Lflag uint32 + Cc [20]byte + Ispeed uint32 + Ospeed uint32 +} + +/* +TurnOnRawIO sets flags suitable for raw I/O (no echo, per-character input, etc) +and returns original flags. +*/ +func TurnOnRawIO(tty *os.File) (orig Termios, err error) { + termios, err := TcGetAttr(tty.Fd()) + if err != nil { + return Termios{}, errors.New("TurnOnRawIO: failed to get flags: " + err.Error()) + } + termiosOrig := *termios + + termios.Lflag &^= syscall.ECHO + termios.Lflag &^= syscall.ICANON + termios.Iflag &^= syscall.IXON + termios.Lflag &^= syscall.ISIG + termios.Iflag |= syscall.IUTF8 + err = TcSetAttr(tty.Fd(), termios) + if err != nil { + return Termios{}, errors.New("TurnOnRawIO: flags to set flags: " + err.Error()) + } + return termiosOrig, nil +} + +func TcSetAttr(fd uintptr, termios *Termios) error { + _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCSETS, uintptr(unsafe.Pointer(termios))) + if err != 0 { + return err + } + return nil +} + +func TcGetAttr(fd uintptr) (*Termios, error) { + termios := &Termios{} + _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(termios))) + if err != 0 { + return nil, err + } + return termios, nil +} diff --git a/internal/cli/clitools/termios_stub.go b/internal/cli/clitools/termios_stub.go new file mode 100644 index 0000000..03397fa --- /dev/null +++ b/internal/cli/clitools/termios_stub.go @@ -0,0 +1,49 @@ +//go:build !linux +// +build !linux + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package clitools + +import ( + "errors" + "os" +) + +type Termios struct { + Iflag uint32 + Oflag uint32 + Cflag uint32 + Lflag uint32 + Cc [20]byte + Ispeed uint32 + Ospeed uint32 +} + +func TurnOnRawIO(tty *os.File) (orig Termios, err error) { + return Termios{}, errors.New("not implemented") +} + +func TcSetAttr(fd uintptr, termios *Termios) error { + return errors.New("not implemented") +} + +func TcGetAttr(fd uintptr) (*Termios, error) { + return nil, errors.New("not implemented") +} diff --git a/internal/cli/ctl/appendlimit.go b/internal/cli/ctl/appendlimit.go new file mode 100644 index 0000000..ce0e3c4 --- /dev/null +++ b/internal/cli/ctl/appendlimit.go @@ -0,0 +1,79 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package ctl + +import ( + "fmt" + + imapbackend "github.com/emersion/go-imap/backend" + "github.com/foxcpp/maddy/framework/module" + "github.com/urfave/cli/v2" +) + +// Copied from go-imap-backend-tests. + +// AppendLimitUser is extension for backend.User interface which allows to +// set append limit value for testing and administration purposes. +type AppendLimitUser interface { + imapbackend.AppendLimitUser + + // SetMessageLimit sets new value for limit. + // nil pointer means no limit. + SetMessageLimit(val *uint32) error +} + +func imapAcctAppendlimit(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + userAL, ok := u.(AppendLimitUser) + if !ok { + return cli.Exit("Error: module.Storage does not support per-user append limit", 2) + } + + if ctx.IsSet("value") { + val := ctx.Int("value") + + var err error + if val == -1 { + err = userAL.SetMessageLimit(nil) + } else { + val32 := uint32(val) + err = userAL.SetMessageLimit(&val32) + } + if err != nil { + return err + } + } else { + lim := userAL.CreateMessageLimit() + if lim == nil { + fmt.Println("No limit") + } else { + fmt.Println(*lim) + } + } + + return nil +} diff --git a/internal/cli/ctl/hash.go b/internal/cli/ctl/hash.go new file mode 100644 index 0000000..5effdf7 --- /dev/null +++ b/internal/cli/ctl/hash.go @@ -0,0 +1,139 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package ctl + +import ( + "fmt" + "os" + "strings" + + "github.com/foxcpp/maddy/internal/auth/pass_table" + maddycli "github.com/foxcpp/maddy/internal/cli" + clitools2 "github.com/foxcpp/maddy/internal/cli/clitools" + "github.com/urfave/cli/v2" + "golang.org/x/crypto/bcrypt" +) + +func init() { + maddycli.AddSubcommand( + &cli.Command{ + Name: "hash", + Usage: "Generate password hashes for use with pass_table", + Action: hashCommand, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "password", + Aliases: []string{"p"}, + Usage: "Use `PASSWORD instead of reading password from stdin\n\t\tWARNING: Provided only for debugging convenience. Don't leave your passwords in shell history!", + }, + &cli.StringFlag{ + Name: "hash", + Usage: "Use specified hash algorithm", + Value: "bcrypt", + }, + &cli.IntFlag{ + Name: "bcrypt-cost", + Usage: "Specify bcrypt cost value", + Value: bcrypt.DefaultCost, + }, + &cli.IntFlag{ + Name: "argon2-time", + Usage: "Time factor for Argon2id", + Value: 3, + }, + &cli.IntFlag{ + Name: "argon2-memory", + Usage: "Memory in KiB to use for Argon2id", + Value: 1024, + }, + &cli.IntFlag{ + Name: "argon2-threads", + Usage: "Threads to use for Argon2id", + Value: 1, + }, + }, + }) +} + +func hashCommand(ctx *cli.Context) error { + hashFunc := ctx.String("hash") + if hashFunc == "" { + hashFunc = pass_table.DefaultHash + } + + hashCompute := pass_table.HashCompute[hashFunc] + if hashCompute == nil { + var funcs []string + for k := range pass_table.HashCompute { + funcs = append(funcs, k) + } + + return cli.Exit(fmt.Sprintf("Error: Unknown hash function, available: %s", strings.Join(funcs, ", ")), 2) + } + + opts := pass_table.HashOpts{ + BcryptCost: bcrypt.DefaultCost, + Argon2Memory: 1024, + Argon2Time: 2, + Argon2Threads: 1, + } + if ctx.IsSet("bcrypt-cost") { + if ctx.Int("bcrypt-cost") > bcrypt.MaxCost { + return cli.Exit("Error: too big bcrypt cost", 2) + } + if ctx.Int("bcrypt-cost") < bcrypt.MinCost { + return cli.Exit("Error: too small bcrypt cost", 2) + } + opts.BcryptCost = ctx.Int("bcrypt-cost") + } + if ctx.IsSet("argon2-memory") { + opts.Argon2Memory = uint32(ctx.Int("argon2-memory")) + } + if ctx.IsSet("argon2-time") { + opts.Argon2Time = uint32(ctx.Int("argon2-time")) + } + if ctx.IsSet("argon2-threads") { + opts.Argon2Threads = uint8(ctx.Int("argon2-threads")) + } + + var pass string + if ctx.IsSet("password") { + pass = ctx.String("password") + } else { + var err error + pass, err = clitools2.ReadPassword("Password") + if err != nil { + return err + } + } + + if pass == "" { + fmt.Fprintln(os.Stderr, "WARNING: This is the hash of an empty string") + } + if strings.TrimSpace(pass) != pass { + fmt.Fprintln(os.Stderr, "WARNING: There is leading/trailing whitespace in the string") + } + + hash, err := hashCompute(opts, pass) + if err != nil { + return err + } + fmt.Println(hashFunc + ":" + hash) + return nil +} diff --git a/internal/cli/ctl/imap.go b/internal/cli/ctl/imap.go new file mode 100644 index 0000000..ea8f820 --- /dev/null +++ b/internal/cli/ctl/imap.go @@ -0,0 +1,893 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package ctl + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/emersion/go-imap" + imapsql "github.com/foxcpp/go-imap-sql" + "github.com/foxcpp/maddy/framework/module" + maddycli "github.com/foxcpp/maddy/internal/cli" + clitools2 "github.com/foxcpp/maddy/internal/cli/clitools" + "github.com/urfave/cli/v2" +) + +func init() { + maddycli.AddSubcommand( + &cli.Command{ + Name: "imap-mboxes", + Usage: "IMAP mailboxes (folders) management", + Subcommands: []*cli.Command{ + { + Name: "list", + Usage: "Show mailboxes of user", + ArgsUsage: "USERNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "subscribed", + Aliases: []string{"s"}, + Usage: "List only subscribed mailboxes", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return mboxesList(be, ctx) + }, + }, + { + Name: "create", + Usage: "Create mailbox", + ArgsUsage: "USERNAME NAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.StringFlag{ + Name: "special", + Usage: "Set SPECIAL-USE attribute on mailbox; valid values: archive, drafts, junk, sent, trash", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return mboxesCreate(be, ctx) + }, + }, + { + Name: "remove", + Usage: "Remove mailbox", + Description: "WARNING: All contents of mailbox will be irrecoverably lost.", + ArgsUsage: "USERNAME MAILBOX", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "yes", + Aliases: []string{"y"}, + Usage: "Don't ask for confirmation", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return mboxesRemove(be, ctx) + }, + }, + { + Name: "rename", + Usage: "Rename mailbox", + Description: "Rename may cause unexpected failures on client-side so be careful.", + ArgsUsage: "USERNAME OLDNAME NEWNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return mboxesRename(be, ctx) + }, + }, + }, + }) + maddycli.AddSubcommand(&cli.Command{ + Name: "imap-msgs", + Usage: "IMAP messages management", + Subcommands: []*cli.Command{ + { + Name: "add", + Usage: "Add message to mailbox", + ArgsUsage: "USERNAME MAILBOX", + Description: "Reads message body (with headers) from stdin. Prints UID of created message on success.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.StringSliceFlag{ + Name: "flag", + Aliases: []string{"f"}, + Usage: "Add flag to message. Can be specified multiple times", + }, + &cli.TimestampFlag{ + Layout: time.RFC3339, + Name: "date", + Aliases: []string{"d"}, + Usage: "Set internal date value to specified one in ISO 8601 format (2006-01-02T15:04:05Z07:00)", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsAdd(be, ctx) + }, + }, + { + Name: "add-flags", + Usage: "Add flags to messages", + ArgsUsage: "USERNAME MAILBOX SEQ FLAGS...", + Description: "Add flags to all messages matched by SEQ.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQSET instead of sequence numbers", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsFlags(be, ctx) + }, + }, + { + Name: "rem-flags", + Usage: "Remove flags from messages", + ArgsUsage: "USERNAME MAILBOX SEQ FLAGS...", + Description: "Remove flags from all messages matched by SEQ.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQSET instead of sequence numbers", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsFlags(be, ctx) + }, + }, + { + Name: "set-flags", + Usage: "Set flags on messages", + ArgsUsage: "USERNAME MAILBOX SEQ FLAGS...", + Description: "Set flags on all messages matched by SEQ.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQSET instead of sequence numbers", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsFlags(be, ctx) + }, + }, + { + Name: "remove", + Usage: "Remove messages from mailbox", + ArgsUsage: "USERNAME MAILBOX SEQSET", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid,u", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQSET instead of sequence numbers", + }, + &cli.BoolFlag{ + Name: "yes", + Aliases: []string{"y"}, + Usage: "Don't ask for confirmation", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsRemove(be, ctx) + }, + }, + { + Name: "copy", + Usage: "Copy messages between mailboxes", + Description: "Note: You can't copy between mailboxes of different users. APPENDLIMIT of target mailbox is not enforced.", + ArgsUsage: "USERNAME SRCMAILBOX SEQSET TGTMAILBOX", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQSET instead of sequence numbers", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsCopy(be, ctx) + }, + }, + { + Name: "move", + Usage: "Move messages between mailboxes", + Description: "Note: You can't move between mailboxes of different users. APPENDLIMIT of target mailbox is not enforced.", + ArgsUsage: "USERNAME SRCMAILBOX SEQSET TGTMAILBOX", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQSET instead of sequence numbers", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsMove(be, ctx) + }, + }, + { + Name: "list", + Usage: "List messages in mailbox", + Description: "If SEQSET is specified - only show messages that match it.", + ArgsUsage: "USERNAME MAILBOX [SEQSET]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQSET instead of sequence numbers", + }, + &cli.BoolFlag{ + Name: "full,f", + Aliases: []string{"f"}, + Usage: "Show entire envelope and all server meta-data", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsList(be, ctx) + }, + }, + { + Name: "dump", + Usage: "Dump message body", + Description: "If passed SEQ matches multiple messages - they will be joined.", + ArgsUsage: "USERNAME MAILBOX SEQ", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQ instead of sequence numbers", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsDump(be, ctx) + }, + }, + }, + }) +} + +func FormatAddress(addr *imap.Address) string { + return fmt.Sprintf("%s <%s@%s>", addr.PersonalName, addr.MailboxName, addr.HostName) +} + +func FormatAddressList(addrs []*imap.Address) string { + res := make([]string, 0, len(addrs)) + for _, addr := range addrs { + res = append(res, FormatAddress(addr)) + } + return strings.Join(res, ", ") +} + +func mboxesList(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + mboxes, err := u.ListMailboxes(ctx.Bool("subscribed,s")) + if err != nil { + return err + } + + if len(mboxes) == 0 && !ctx.Bool("quiet") { + fmt.Fprintln(os.Stderr, "No mailboxes.") + } + + for _, info := range mboxes { + if len(info.Attributes) != 0 { + fmt.Print(info.Name, "\t", info.Attributes, "\n") + } else { + fmt.Println(info.Name) + } + } + + return nil +} + +func mboxesCreate(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + name := ctx.Args().Get(1) + if name == "" { + return cli.Exit("Error: NAME is required", 2) + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + if ctx.IsSet("special") { + attr := "\\" + strings.Title(ctx.String("special")) //nolint:staticcheck + // (nolint) strings.Title is perfectly fine there since special mailbox tags will never use Unicode. + + suu, ok := u.(SpecialUseUser) + if !ok { + return cli.Exit("Error: storage backend does not support SPECIAL-USE IMAP extension", 2) + } + + return suu.CreateMailboxSpecial(name, attr) + } + + return u.CreateMailbox(name) +} + +func mboxesRemove(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + name := ctx.Args().Get(1) + if name == "" { + return cli.Exit("Error: NAME is required", 2) + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + if !ctx.Bool("yes") { + status, err := u.Status(name, []imap.StatusItem{imap.StatusMessages}) + if err != nil { + return err + } + + if status.Messages != 0 { + fmt.Fprintf(os.Stderr, "Mailbox %s contains %d messages.\n", name, status.Messages) + } + + if !clitools2.Confirmation("Are you sure you want to delete that mailbox?", false) { + return errors.New("Cancelled") + } + } + + return u.DeleteMailbox(name) +} + +func mboxesRename(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + oldName := ctx.Args().Get(1) + if oldName == "" { + return cli.Exit("Error: OLDNAME is required", 2) + } + newName := ctx.Args().Get(2) + if newName == "" { + return cli.Exit("Error: NEWNAME is required", 2) + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + return u.RenameMailbox(oldName, newName) +} + +func msgsAdd(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + name := ctx.Args().Get(1) + if name == "" { + return cli.Exit("Error: MAILBOX is required", 2) + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + flags := ctx.StringSlice("flag") + if flags == nil { + flags = []string{} + } + + date := time.Now() + if ctx.IsSet("date") { + date = *ctx.Timestamp("date") + } + + buf := bytes.Buffer{} + if _, err := io.Copy(&buf, os.Stdin); err != nil { + return err + } + + if buf.Len() == 0 { + return cli.Exit("Error: Empty message, refusing to continue", 2) + } + + status, err := u.Status(name, []imap.StatusItem{imap.StatusUidNext}) + if err != nil { + return err + } + + if err := u.CreateMessage(name, flags, date, &buf, nil); err != nil { + return err + } + + // TODO: Use APPENDUID + fmt.Println(status.UidNext) + + return nil +} + +func msgsRemove(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + name := ctx.Args().Get(1) + if name == "" { + return cli.Exit("Error: MAILBOX is required", 2) + } + seqset := ctx.Args().Get(2) + if seqset == "" { + return cli.Exit("Error: SEQSET is required", 2) + } + + if !ctx.Bool("uid") { + fmt.Fprintln(os.Stderr, "WARNING: --uid=true will be the default in 0.7") + } + + seq, err := imap.ParseSeqSet(seqset) + if err != nil { + return err + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + _, mbox, err := u.GetMailbox(name, true, nil) + if err != nil { + return err + } + + if !ctx.Bool("yes") { + if !clitools2.Confirmation("Are you sure you want to delete these messages?", false) { + return errors.New("Cancelled") + } + } + + mboxB := mbox.(*imapsql.Mailbox) + return mboxB.DelMessages(ctx.Bool("uid"), seq) +} + +func msgsCopy(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + srcName := ctx.Args().Get(1) + if srcName == "" { + return cli.Exit("Error: SRCMAILBOX is required", 2) + } + seqset := ctx.Args().Get(2) + if seqset == "" { + return cli.Exit("Error: SEQSET is required", 2) + } + tgtName := ctx.Args().Get(3) + if tgtName == "" { + return cli.Exit("Error: TGTMAILBOX is required", 2) + } + + if !ctx.Bool("uid") { + fmt.Fprintln(os.Stderr, "WARNING: --uid=true will be the default in 0.7") + } + + seq, err := imap.ParseSeqSet(seqset) + if err != nil { + return err + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + _, srcMbox, err := u.GetMailbox(srcName, true, nil) + if err != nil { + return err + } + + return srcMbox.CopyMessages(ctx.Bool("uid"), seq, tgtName) +} + +func msgsMove(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + srcName := ctx.Args().Get(1) + if srcName == "" { + return cli.Exit("Error: SRCMAILBOX is required", 2) + } + seqset := ctx.Args().Get(2) + if seqset == "" { + return cli.Exit("Error: SEQSET is required", 2) + } + tgtName := ctx.Args().Get(3) + if tgtName == "" { + return cli.Exit("Error: TGTMAILBOX is required", 2) + } + + if !ctx.Bool("uid") { + fmt.Fprintln(os.Stderr, "WARNING: --uid=true will be the default in 0.7") + } + + seq, err := imap.ParseSeqSet(seqset) + if err != nil { + return err + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + _, srcMbox, err := u.GetMailbox(srcName, true, nil) + if err != nil { + return err + } + + moveMbox := srcMbox.(*imapsql.Mailbox) + + return moveMbox.MoveMessages(ctx.Bool("uid"), seq, tgtName) +} + +func msgsList(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + mboxName := ctx.Args().Get(1) + if mboxName == "" { + return cli.Exit("Error: MAILBOX is required", 2) + } + seqset := ctx.Args().Get(2) + uid := ctx.Bool("uid") + if seqset == "" { + seqset = "1:*" + uid = true + } else if !uid { + fmt.Fprintln(os.Stderr, "WARNING: --uid=true will be the default in 0.7") + } + + seq, err := imap.ParseSeqSet(seqset) + if err != nil { + return err + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + _, mbox, err := u.GetMailbox(mboxName, true, nil) + if err != nil { + return err + } + + ch := make(chan *imap.Message, 10) + go func() { + err = mbox.ListMessages(uid, seq, []imap.FetchItem{imap.FetchEnvelope, imap.FetchInternalDate, imap.FetchRFC822Size, imap.FetchFlags, imap.FetchUid}, ch) + }() + + for msg := range ch { + if !ctx.Bool("full") { + fmt.Printf("UID %d: %s - %s\n %v, %v\n\n", msg.Uid, FormatAddressList(msg.Envelope.From), msg.Envelope.Subject, msg.Flags, msg.Envelope.Date) + continue + } + + fmt.Println("- Server meta-data:") + fmt.Println("UID:", msg.Uid) + fmt.Println("Sequence number:", msg.SeqNum) + fmt.Println("Flags:", msg.Flags) + fmt.Println("Body size:", msg.Size) + fmt.Println("Internal date:", msg.InternalDate.Unix(), msg.InternalDate) + fmt.Println("- Envelope:") + if len(msg.Envelope.From) != 0 { + fmt.Println("From:", FormatAddressList(msg.Envelope.From)) + } + if len(msg.Envelope.To) != 0 { + fmt.Println("To:", FormatAddressList(msg.Envelope.To)) + } + if len(msg.Envelope.Cc) != 0 { + fmt.Println("CC:", FormatAddressList(msg.Envelope.Cc)) + } + if len(msg.Envelope.Bcc) != 0 { + fmt.Println("BCC:", FormatAddressList(msg.Envelope.Bcc)) + } + if msg.Envelope.InReplyTo != "" { + fmt.Println("In-Reply-To:", msg.Envelope.InReplyTo) + } + if msg.Envelope.MessageId != "" { + fmt.Println("Message-Id:", msg.Envelope.MessageId) + } + if !msg.Envelope.Date.IsZero() { + fmt.Println("Date:", msg.Envelope.Date.Unix(), msg.Envelope.Date) + } + if msg.Envelope.Subject != "" { + fmt.Println("Subject:", msg.Envelope.Subject) + } + fmt.Println() + } + return err +} + +func msgsDump(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + mboxName := ctx.Args().Get(1) + if mboxName == "" { + return cli.Exit("Error: MAILBOX is required", 2) + } + seqset := ctx.Args().Get(2) + uid := ctx.Bool("uid") + if seqset == "" { + seqset = "1:*" + uid = true + } else if !uid { + fmt.Fprintln(os.Stderr, "WARNING: --uid=true will be the default in 0.7") + } + + seq, err := imap.ParseSeqSet(seqset) + if err != nil { + return err + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + _, mbox, err := u.GetMailbox(mboxName, true, nil) + if err != nil { + return err + } + + ch := make(chan *imap.Message, 10) + go func() { + err = mbox.ListMessages(uid, seq, []imap.FetchItem{imap.FetchRFC822}, ch) + }() + + for msg := range ch { + for _, v := range msg.Body { + if _, err := io.Copy(os.Stdout, v); err != nil { + return err + } + } + } + return err +} + +func msgsFlags(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + name := ctx.Args().Get(1) + if name == "" { + return cli.Exit("Error: MAILBOX is required", 2) + } + seqStr := ctx.Args().Get(2) + if seqStr == "" { + return cli.Exit("Error: SEQ is required", 2) + } + + if !ctx.Bool("uid") { + fmt.Fprintln(os.Stderr, "WARNING: --uid=true will be the default in 0.7") + } + + seq, err := imap.ParseSeqSet(seqStr) + if err != nil { + return err + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + _, mbox, err := u.GetMailbox(name, false, nil) + if err != nil { + return err + } + + flags := ctx.Args().Slice()[3:] + if len(flags) == 0 { + return cli.Exit("Error: at least once FLAG is required", 2) + } + + var op imap.FlagsOp + switch ctx.Command.Name { + case "add-flags": + op = imap.AddFlags + case "rem-flags": + op = imap.RemoveFlags + case "set-flags": + op = imap.SetFlags + default: + panic("unknown command: " + ctx.Command.Name) + } + + return mbox.UpdateMessagesFlags(ctx.Bool("uid"), seq, op, true, flags) +} diff --git a/internal/cli/ctl/imapacct.go b/internal/cli/ctl/imapacct.go new file mode 100644 index 0000000..2541a22 --- /dev/null +++ b/internal/cli/ctl/imapacct.go @@ -0,0 +1,301 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package ctl + +import ( + "errors" + "fmt" + "os" + + "github.com/emersion/go-imap" + "github.com/foxcpp/maddy/framework/module" + maddycli "github.com/foxcpp/maddy/internal/cli" + clitools2 "github.com/foxcpp/maddy/internal/cli/clitools" + "github.com/urfave/cli/v2" +) + +func init() { + maddycli.AddSubcommand( + &cli.Command{ + Name: "imap-acct", + Usage: "IMAP storage accounts management", + Description: `These subcommands can be used to list/create/delete IMAP storage +accounts for any storage backend supported by maddy. + +The corresponding storage backend should be configured in maddy.conf and be +defined in a top-level configuration block. By default, the name of that +block should be local_mailboxes but this can be changed using --cfg-block +flag for subcommands. + +Note that in default configuration it is not enough to create an IMAP storage +account to grant server access. Additionally, user credentials should +be created using 'creds' subcommand. +`, + Subcommands: []*cli.Command{ + { + Name: "list", + Usage: "List storage accounts", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return imapAcctList(be, ctx) + }, + }, + { + Name: "create", + Usage: "Create IMAP storage account", + Description: `In addition to account creation, this command +creates a set of default folder (mailboxes) with special-use attribute set.`, + ArgsUsage: "USERNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "no-specialuse", + Usage: "Do not create special-use folders", + Value: false, + }, + &cli.StringFlag{ + Name: "sent-name", + Usage: "Name of special mailbox for sent messages, use empty string to not create any", + Value: "Sent", + }, + &cli.StringFlag{ + Name: "trash-name", + Usage: "Name of special mailbox for trash, use empty string to not create any", + Value: "Trash", + }, + &cli.StringFlag{ + Name: "junk-name", + Usage: "Name of special mailbox for 'junk' (spam), use empty string to not create any", + Value: "Junk", + }, + &cli.StringFlag{ + Name: "drafts-name", + Usage: "Name of special mailbox for drafts, use empty string to not create any", + Value: "Drafts", + }, + &cli.StringFlag{ + Name: "archive-name", + Usage: "Name of special mailbox for archive, use empty string to not create any", + Value: "Archive", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return imapAcctCreate(be, ctx) + }, + }, + { + Name: "remove", + Usage: "Delete IMAP storage account", + Description: `If IMAP connections are open and using the specified account, +messages access will be killed off immediately though connection will remain open. No cache +or other buffering takes effect.`, + ArgsUsage: "USERNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "yes", + Aliases: []string{"y"}, + Usage: "Don't ask for confirmation", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return imapAcctRemove(be, ctx) + }, + }, + { + Name: "appendlimit", + Usage: "Query or set accounts's APPENDLIMIT value", + Description: `APPENDLIMIT value determines the size of a message that +can be saved into a mailbox using IMAP APPEND command. This does not affect the size +of messages that can be delivered to the mailbox from non-IMAP sources (e.g. SMTP). + +Global APPENDLIMIT value set via server configuration takes precedence over +per-account values configured using this command. + +APPENDLIMIT value (either global or per-account) cannot be larger than +4 GiB due to IMAP protocol limitations. +`, + ArgsUsage: "USERNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.IntFlag{ + Name: "value", + Aliases: []string{"v"}, + Usage: "Set APPENDLIMIT to specified value (in bytes)", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return imapAcctAppendlimit(be, ctx) + }, + }, + }, + }) +} + +type SpecialUseUser interface { + CreateMailboxSpecial(name, specialUseAttr string) error +} + +func imapAcctList(be module.Storage, ctx *cli.Context) error { + mbe, ok := be.(module.ManageableStorage) + if !ok { + return cli.Exit("Error: storage backend does not support accounts management using maddy command", 2) + } + + list, err := mbe.ListIMAPAccts() + if err != nil { + return err + } + + if len(list) == 0 && !ctx.Bool("quiet") { + fmt.Fprintln(os.Stderr, "No users.") + } + + for _, user := range list { + fmt.Println(user) + } + return nil +} + +func imapAcctCreate(be module.Storage, ctx *cli.Context) error { + mbe, ok := be.(module.ManageableStorage) + if !ok { + return cli.Exit("Error: storage backend does not support accounts management using maddy command", 2) + } + + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + + if err := mbe.CreateIMAPAcct(username); err != nil { + return err + } + + act, err := mbe.GetIMAPAcct(username) + if err != nil { + return fmt.Errorf("failed to get user: %w", err) + } + + suu, ok := act.(SpecialUseUser) + if !ok { + fmt.Fprintf(os.Stderr, "Note: Storage backend does not support SPECIAL-USE IMAP extension") + } + + if ctx.Bool("no-specialuse") { + return nil + } + + createMbox := func(name, specialUseAttr string) error { + if suu == nil { + return act.CreateMailbox(name) + } + return suu.CreateMailboxSpecial(name, specialUseAttr) + } + + if name := ctx.String("sent-name"); name != "" { + if err := createMbox(name, imap.SentAttr); err != nil { + fmt.Fprintf(os.Stderr, "Failed to create sent folder: %v", err) + } + } + if name := ctx.String("trash-name"); name != "" { + if err := createMbox(name, imap.TrashAttr); err != nil { + fmt.Fprintf(os.Stderr, "Failed to create trash folder: %v", err) + } + } + if name := ctx.String("junk-name"); name != "" { + if err := createMbox(name, imap.JunkAttr); err != nil { + fmt.Fprintf(os.Stderr, "Failed to create junk folder: %v", err) + } + } + if name := ctx.String("drafts-name"); name != "" { + if err := createMbox(name, imap.DraftsAttr); err != nil { + fmt.Fprintf(os.Stderr, "Failed to create drafts folder: %v", err) + } + } + if name := ctx.String("archive-name"); name != "" { + if err := createMbox(name, imap.ArchiveAttr); err != nil { + fmt.Fprintf(os.Stderr, "Failed to create archive folder: %v", err) + } + } + + return nil +} + +func imapAcctRemove(be module.Storage, ctx *cli.Context) error { + mbe, ok := be.(module.ManageableStorage) + if !ok { + return cli.Exit("Error: storage backend does not support accounts management using maddy command", 2) + } + + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + + if !ctx.Bool("yes") { + if !clitools2.Confirmation("Are you sure you want to delete this user account?", false) { + return errors.New("Cancelled") + } + } + + return mbe.DeleteIMAPAcct(username) +} diff --git a/internal/cli/ctl/moduleinit.go b/internal/cli/ctl/moduleinit.go new file mode 100644 index 0000000..23e79e5 --- /dev/null +++ b/internal/cli/ctl/moduleinit.go @@ -0,0 +1,133 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package ctl + +import ( + "errors" + "fmt" + "io" + "os" + + "github.com/foxcpp/maddy" + parser "github.com/foxcpp/maddy/framework/cfgparser" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/updatepipe" + "github.com/urfave/cli/v2" +) + +func closeIfNeeded(i interface{}) { + if c, ok := i.(io.Closer); ok { + c.Close() + } +} + +func getCfgBlockModule(ctx *cli.Context) (map[string]interface{}, *maddy.ModInfo, error) { + cfgPath := ctx.String("config") + if cfgPath == "" { + return nil, nil, cli.Exit("Error: config is required", 2) + } + cfgFile, err := os.Open(cfgPath) + if err != nil { + return nil, nil, cli.Exit(fmt.Sprintf("Error: failed to open config: %v", err), 2) + } + defer cfgFile.Close() + cfgNodes, err := parser.Read(cfgFile, cfgFile.Name()) + if err != nil { + return nil, nil, cli.Exit(fmt.Sprintf("Error: failed to parse config: %v", err), 2) + } + + globals, cfgNodes, err := maddy.ReadGlobals(cfgNodes) + if err != nil { + return nil, nil, err + } + + if err := maddy.InitDirs(); err != nil { + return nil, nil, err + } + + module.NoRun = true + _, mods, err := maddy.RegisterModules(globals, cfgNodes) + if err != nil { + return nil, nil, err + } + defer hooks.RunHooks(hooks.EventShutdown) + + cfgBlock := ctx.String("cfg-block") + if cfgBlock == "" { + return nil, nil, cli.Exit("Error: cfg-block is required", 2) + } + var mod maddy.ModInfo + for _, m := range mods { + if m.Instance.InstanceName() == cfgBlock { + mod = m + break + } + } + if mod.Instance == nil { + return nil, nil, cli.Exit(fmt.Sprintf("Error: unknown configuration block: %s", cfgBlock), 2) + } + + return globals, &mod, nil +} + +func openStorage(ctx *cli.Context) (module.Storage, error) { + globals, mod, err := getCfgBlockModule(ctx) + if err != nil { + return nil, err + } + + storage, ok := mod.Instance.(module.Storage) + if !ok { + return nil, cli.Exit(fmt.Sprintf("Error: configuration block %s is not an IMAP storage", ctx.String("cfg-block")), 2) + } + + if err := mod.Instance.Init(config.NewMap(globals, mod.Cfg)); err != nil { + return nil, fmt.Errorf("Error: module initialization failed: %w", err) + } + + if updStore, ok := mod.Instance.(updatepipe.Backend); ok { + if err := updStore.EnableUpdatePipe(updatepipe.ModePush); err != nil && !errors.Is(err, os.ErrNotExist) { + fmt.Fprintf(os.Stderr, "Failed to initialize update pipe, do not remove messages from mailboxes open by clients: %v\n", err) + } + } else { + fmt.Fprintf(os.Stderr, "No update pipe support, do not remove messages from mailboxes open by clients\n") + } + + return storage, nil +} + +func openUserDB(ctx *cli.Context) (module.PlainUserDB, error) { + globals, mod, err := getCfgBlockModule(ctx) + if err != nil { + return nil, err + } + + userDB, ok := mod.Instance.(module.PlainUserDB) + if !ok { + return nil, cli.Exit(fmt.Sprintf("Error: configuration block %s is not a local credentials store", ctx.String("cfg-block")), 2) + } + + if err := mod.Instance.Init(config.NewMap(globals, mod.Cfg)); err != nil { + return nil, fmt.Errorf("Error: module initialization failed: %w", err) + } + + return userDB, nil +} diff --git a/internal/cli/ctl/users.go b/internal/cli/ctl/users.go new file mode 100644 index 0000000..09dc909 --- /dev/null +++ b/internal/cli/ctl/users.go @@ -0,0 +1,246 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package ctl + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth/pass_table" + maddycli "github.com/foxcpp/maddy/internal/cli" + clitools2 "github.com/foxcpp/maddy/internal/cli/clitools" + "github.com/urfave/cli/v2" + "golang.org/x/crypto/bcrypt" +) + +func init() { + maddycli.AddSubcommand( + &cli.Command{ + Name: "creds", + Usage: "Local credentials management", + Description: `These commands manipulate credential databases used by +maddy mail server. + +Corresponding credential database should be defined in maddy.conf as +a top-level config block. By default the block name should be local_authdb ( +can be changed using --cfg-block argument for subcommands). + +Note that it is not enough to create user credentials in order to grant +IMAP access - IMAP account should be also created using 'imap-acct create' subcommand. +`, + Subcommands: []*cli.Command{ + { + Name: "list", + Usage: "List created credentials", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_authdb", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openUserDB(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return usersList(be, ctx) + }, + }, + { + Name: "create", + Usage: "Create user account", + Description: `Reads password from stdin. + +If configuration block uses auth.pass_table, then hash algorithm can be configured +using command flags. Otherwise, these options cannot be used. +`, + ArgsUsage: "USERNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_authdb", + }, + &cli.StringFlag{ + Name: "password", + Aliases: []string{"p"}, + Usage: "Use `PASSWORD instead of reading password from stdin.\n\t\tWARNING: Provided only for debugging convenience. Don't leave your passwords in shell history!", + }, + &cli.StringFlag{ + Name: "hash", + Usage: "Use specified hash algorithm. Valid values: " + strings.Join(pass_table.Hashes, ", "), + Value: "bcrypt", + }, + &cli.IntFlag{ + Name: "bcrypt-cost", + Usage: "Specify bcrypt cost value", + Value: bcrypt.DefaultCost, + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openUserDB(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return usersCreate(be, ctx) + }, + }, + { + Name: "remove", + Usage: "Delete user account", + ArgsUsage: "USERNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_authdb", + }, + &cli.BoolFlag{ + Name: "yes", + Aliases: []string{"y"}, + Usage: "Don't ask for confirmation", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openUserDB(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return usersRemove(be, ctx) + }, + }, + { + Name: "password", + Usage: "Change account password", + Description: "Reads password from stdin", + ArgsUsage: "USERNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_authdb", + }, + &cli.StringFlag{ + Name: "password", + Aliases: []string{"p"}, + Usage: "Use `PASSWORD` instead of reading password from stdin.\n\t\tWARNING: Provided only for debugging convenience. Don't leave your passwords in shell history!", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openUserDB(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return usersPassword(be, ctx) + }, + }, + }, + }) +} + +func usersList(be module.PlainUserDB, ctx *cli.Context) error { + list, err := be.ListUsers() + if err != nil { + return err + } + + if len(list) == 0 && !ctx.Bool("quiet") { + fmt.Fprintln(os.Stderr, "No users.") + } + + for _, user := range list { + fmt.Println(user) + } + return nil +} + +func usersCreate(be module.PlainUserDB, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + + var pass string + if ctx.IsSet("password") { + pass = ctx.String("password") + } else { + var err error + pass, err = clitools2.ReadPassword("Enter password for new user") + if err != nil { + return err + } + } + + if beHash, ok := be.(*pass_table.Auth); ok { + return beHash.CreateUserHash(username, pass, ctx.String("hash"), pass_table.HashOpts{ + BcryptCost: ctx.Int("bcrypt-cost"), + }) + } else if ctx.IsSet("hash") || ctx.IsSet("bcrypt-cost") { + return cli.Exit("Error: --hash cannot be used with non-pass_table credentials DB", 2) + } else { + return be.CreateUser(username, pass) + } +} + +func usersRemove(be module.PlainUserDB, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return errors.New("Error: USERNAME is required") + } + + if !ctx.Bool("yes") { + if !clitools2.Confirmation("Are you sure you want to delete this user account?", false) { + return errors.New("Cancelled") + } + } + + return be.DeleteUser(username) +} + +func usersPassword(be module.PlainUserDB, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return errors.New("Error: USERNAME is required") + } + + var pass string + if ctx.IsSet("password") { + pass = ctx.String("password") + } else { + var err error + pass, err = clitools2.ReadPassword("Enter new password") + if err != nil { + return err + } + } + + return be.SetUserPassword(username, pass) +} diff --git a/internal/cli/extflag.go b/internal/cli/extflag.go new file mode 100644 index 0000000..8cfc27c --- /dev/null +++ b/internal/cli/extflag.go @@ -0,0 +1,60 @@ +package maddycli + +import ( + "flag" + + "github.com/urfave/cli/v2" +) + +// extFlag implements cli.Flag via standard flag.Flag. +type extFlag struct { + f *flag.Flag +} + +func (e *extFlag) Apply(fs *flag.FlagSet) error { + fs.Var(e.f.Value, e.f.Name, e.f.Usage) + return nil +} + +func (e *extFlag) Names() []string { + return []string{e.f.Name} +} + +func (e *extFlag) IsSet() bool { + return false +} + +func (e *extFlag) String() string { + return cli.FlagStringer(e) +} + +func (e *extFlag) IsVisible() bool { + return true +} + +func (e *extFlag) TakesValue() bool { + return false +} + +func (e *extFlag) GetUsage() string { + return e.f.Usage +} + +func (e *extFlag) GetValue() string { + return e.f.Value.String() +} + +func (e *extFlag) GetDefaultText() string { + return e.f.DefValue +} + +func (e *extFlag) GetEnvVars() []string { + return nil +} + +func mapStdlibFlags(app *cli.App) { + // Modified AllowExtFlags from cli lib with -test.* exception removed. + flag.VisitAll(func(f *flag.Flag) { + app.Flags = append(app.Flags, &extFlag{f}) + }) +} diff --git a/internal/dmarc/dmarc.go b/internal/dmarc/dmarc.go new file mode 100644 index 0000000..723fedd --- /dev/null +++ b/internal/dmarc/dmarc.go @@ -0,0 +1,42 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dmarc + +import ( + "context" + + "github.com/emersion/go-msgauth/dmarc" +) + +type ( + Resolver interface { + LookupTXT(context.Context, string) ([]string, error) + } + + Record = dmarc.Record + Policy = dmarc.Policy + AlignmentMode = dmarc.AlignmentMode + FailureOptions = dmarc.FailureOptions +) + +const ( + PolicyNone = dmarc.PolicyNone + PolicyReject = dmarc.PolicyReject + PolicyQuarantine = dmarc.PolicyQuarantine +) diff --git a/internal/dmarc/evaluate.go b/internal/dmarc/evaluate.go new file mode 100644 index 0000000..ff978e5 --- /dev/null +++ b/internal/dmarc/evaluate.go @@ -0,0 +1,256 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dmarc + +import ( + "context" + "errors" + "fmt" + "net" + "net/mail" + "strings" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/emersion/go-msgauth/dmarc" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/dns" + "golang.org/x/net/publicsuffix" +) + +// FetchRecord looks up the DMARC record relevant for the RFC5322.From domain. +// It returns the record and the domain it was found with (may not be +// equal to the RFC5322.From domain). +func FetchRecord(ctx context.Context, r Resolver, fromDomain string) (policyDomain string, rec *Record, err error) { + policyDomain = fromDomain + + // 1. Lookup using From Domain. + txts, err := r.LookupTXT(ctx, dns.FQDN("_dmarc."+fromDomain)) + if err != nil { + dnsErr, ok := err.(*net.DNSError) + if !ok || !dnsErr.IsNotFound { + return "", nil, err + } + } + if len(txts) == 0 { + // No records or 'no such host', try orgDomain. + orgDomain, err := publicsuffix.EffectiveTLDPlusOne(fromDomain) + if err != nil { + return "", nil, err + } + + policyDomain = orgDomain + + txts, err = r.LookupTXT(ctx, dns.FQDN("_dmarc."+orgDomain)) + if err != nil { + dnsErr, ok := err.(*net.DNSError) + if !ok || !dnsErr.IsNotFound { + return "", nil, err + } + } + // Still nothing? Bail out. + if len(txts) == 0 { + return "", nil, nil + } + } + + // Exclude records that are not DMARC policies. + records := txts[:0] + for _, txt := range txts { + if strings.HasPrefix(txt, "v=DMARC1") { + records = append(records, txt) + } + } + // Multiple records => no record. + if len(records) > 1 || len(records) == 0 { + return "", nil, nil + } + + rec, err = dmarc.Parse(records[0]) + + return policyDomain, rec, err +} + +type EvalResult struct { + // The Authentication-Results field generated as a result of the DMARC + // check. + Authres authres.DMARCResult + + // The Authentication-Results field for SPF that was considered during + // alignment check. May be empty. + SPFResult authres.SPFResult + + // Whether HELO or MAIL FROM match the RFC5322.From domain. + SPFAligned bool + + // The Authentication-Results field for the DKIM signature that is aligned, + // if no signatures are aligned - this field contains the result for the + // first signature. May be empty. + DKIMResult authres.DKIMResult + + // Whether there is a DKIM signature with the d= field matching the + // RFC5322.From domain. + DKIMAligned bool +} + +// EvaluateAlignment checks whether identifiers authenticated by SPF and DKIM are in alignment +// with the RFC5322.Domain. +// +// It returns EvalResult which contains the Authres field with the actual check result and +// a bunch of other trace information that can be useful for troubleshooting +// (and also report generation). +func EvaluateAlignment(fromDomain string, record *Record, results []authres.Result) EvalResult { + var ( + spfAligned = false + spfResult = authres.SPFResult{} + dkimAligned = false + dkimResult = authres.DKIMResult{} + dkimPresent = false + dkimTempFail = false + ) + for _, res := range results { + if dkimRes, ok := res.(*authres.DKIMResult); ok { + dkimPresent = true + + // We want to return DKIM result for a signature provided by the orgDomain, + // in case there is none - return any (possibly misaligned) for reference. + if dkimResult.Value == "" { + dkimResult = *dkimRes + } + if isAligned(fromDomain, dkimRes.Domain, record.DKIMAlignment) { + dkimResult = *dkimRes + switch dkimRes.Value { + case authres.ResultPass: + dkimAligned = true + case authres.ResultTempError: + dkimTempFail = true + } + } + } + if spfRes, ok := res.(*authres.SPFResult); ok { + spfResult = *spfRes + var aligned bool + if spfRes.From == "" { + aligned = isAligned(fromDomain, spfRes.Helo, record.SPFAlignment) + } else { + aligned = isAligned(fromDomain, spfRes.From, record.SPFAlignment) + } + if aligned && spfRes.Value == authres.ResultPass { + spfAligned = true + } + } + } + + res := EvalResult{ + SPFResult: spfResult, + SPFAligned: spfAligned, + DKIMResult: dkimResult, + DKIMAligned: dkimAligned, + } + + if !dkimPresent || spfResult.Value == "" { + res.Authres = authres.DMARCResult{ + Value: authres.ResultNone, + Reason: "Not enough information (required checks are disabled)", + From: fromDomain, + } + return res + } + + if dkimTempFail && !dkimAligned && !spfAligned { + // We can't be sure whether it is aligned or not. Bail out. + res.Authres = authres.DMARCResult{ + Value: authres.ResultTempError, + Reason: "DKIM authentication temp error", + From: fromDomain, + } + return res + } + if !dkimAligned && spfResult.Value == authres.ResultTempError { + // We can't be sure whether it is aligned or not. Bail out. + res.Authres = authres.DMARCResult{ + Value: authres.ResultTempError, + Reason: "SPF authentication temp error", + From: fromDomain, + } + return res + } + + res.Authres.From = fromDomain + if dkimAligned || spfAligned { + res.Authres.Value = authres.ResultPass + } else { + res.Authres.Value = authres.ResultFail + res.Authres.Reason = "No aligned identifiers" + } + return res +} + +func isAligned(fromDomain, authDomain string, mode AlignmentMode) bool { + if mode == dmarc.AlignmentStrict { + return strings.EqualFold(fromDomain, authDomain) + } + + tld, _ := publicsuffix.PublicSuffix(fromDomain) + if strings.EqualFold(fromDomain, tld) { + return strings.EqualFold(fromDomain, authDomain) + } + orgDomainFrom, err := publicsuffix.EffectiveTLDPlusOne(fromDomain) + if err != nil { + return false + } + authDomainFrom, err := publicsuffix.EffectiveTLDPlusOne(authDomain) + if err != nil { + return false + } + + return strings.EqualFold(orgDomainFrom, authDomainFrom) +} + +func ExtractFromDomain(hdr textproto.Header) (string, error) { + // TODO(GH emersion/go-message#75): Add textproto.Header.Count method. + var firstFrom string + for fields := hdr.FieldsByKey("From"); fields.Next(); { + if firstFrom == "" { + firstFrom = fields.Value() + } else { + return "", errors.New("dmarc: multiple From header fields are not allowed") + } + } + if firstFrom == "" { + return "", errors.New("dmarc: missing From header field") + } + + hdrFromList, err := mail.ParseAddressList(firstFrom) + if err != nil { + return "", fmt.Errorf("dmarc: malformed From header field: %s", strings.TrimPrefix(err.Error(), "mail: ")) + } + if len(hdrFromList) > 1 { + return "", errors.New("dmarc: multiple addresses in From field are not allowed") + } + if len(hdrFromList) == 0 { + return "", errors.New("dmarc: missing address in From field") + } + _, domain, err := address.Split(hdrFromList[0].Address) + if err != nil { + return "", fmt.Errorf("dmarc: malformed From header field: %w", err) + } + + return domain, nil +} diff --git a/internal/dmarc/evaluate_test.go b/internal/dmarc/evaluate_test.go new file mode 100644 index 0000000..c44bbbe --- /dev/null +++ b/internal/dmarc/evaluate_test.go @@ -0,0 +1,514 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dmarc + +import ( + "bufio" + "strings" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/emersion/go-msgauth/dmarc" +) + +func TestEvaluateAlignment(t *testing.T) { + type tCase struct { + fromDomain string + record *Record + results []authres.Result + + output authres.ResultValue + } + test := func(i int, c tCase) { + out := EvaluateAlignment(c.fromDomain, c.record, c.results) + t.Logf("%d - %+v", i, out) + if out.Authres.Value != c.output { + t.Errorf("%d: Wrong eval result, want '%s', got '%s' (%+v)", i, c.output, out.Authres.Value, out) + } + } + + cases := []tCase{ + { // 0 + fromDomain: "example.org", + record: &Record{}, + + output: authres.ResultNone, + }, + { // 1 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultFail, + From: "example.org", + Helo: "mx.example.org", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultFail, + }, + { // 2 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "example.org", + Helo: "mx.example.org", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultPass, + }, + { // 3 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultFail, + From: "example.org", + Helo: "mx.example.org", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultFail, + }, + { // 4 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "example.com", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultFail, + }, + { // 5 + fromDomain: "example.com", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "cbg.bounces.example.com", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultPass, + }, + { // 6 + fromDomain: "example.com", + record: &Record{ + SPFAlignment: dmarc.AlignmentStrict, + }, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "cbg.bounces.example.com", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultFail, + }, + { // 7 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultFail, + Domain: "example.org", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultFail, + }, + { // 8 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "example.org", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultPass, + }, + { // 9 + fromDomain: "example.com", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "cbg.bounces.example.com", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "example.com", + }, + }, + output: authres.ResultPass, + }, + { // 10 + fromDomain: "example.com", + record: &Record{ + SPFAlignment: dmarc.AlignmentRelaxed, + }, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "cbg.bounces.example.com", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultFail, + Domain: "example.com", + }, + }, + output: authres.ResultPass, + }, + { // 11 + fromDomain: "example.com", + record: &Record{ + SPFAlignment: dmarc.AlignmentStrict, + }, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "cbg.bounces.example.com", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "example.com", + }, + }, + output: authres.ResultPass, + }, + { // 12 + fromDomain: "example.com", + record: &Record{ + SPFAlignment: dmarc.AlignmentStrict, + DKIMAlignment: dmarc.AlignmentStrict, + }, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "cbg.bounces.example.com", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultFail, + Domain: "cbg.example.com", + }, + }, + output: authres.ResultFail, + }, + { // 13 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultFail, + Domain: "example.org", + }, + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "example.net", + }, + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "example.org", + }, + &authres.DKIMResult{ + Value: authres.ResultFail, + Domain: "example.com", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultPass, + }, + { // 14 + fromDomain: "example.com", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultPass, + }, + { // 15 + fromDomain: "example.com", + record: &Record{ + SPFAlignment: dmarc.AlignmentStrict, + }, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultFail, + }, + { // 16 + fromDomain: "example.com", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultTempError, + From: "", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultTempError, + }, + { // 17 + fromDomain: "example.com", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultTempError, + Domain: "example.com", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultTempError, + }, + { // 18 + fromDomain: "example.com", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultTempError, + From: "", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "example.com", + }, + }, + output: authres.ResultPass, + }, + { // 19 + fromDomain: "example.com", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultTempError, + Domain: "example.com", + }, + }, + output: authres.ResultPass, + }, + { // 20 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "example.org", + }, + &authres.DKIMResult{ + Value: authres.ResultTempError, + Domain: "example.org", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultPass, + }, + { // 21 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultFail, + Domain: "example.org", + }, + &authres.DKIMResult{ + Value: authres.ResultTempError, + Domain: "example.org", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultTempError, + }, + { // 22 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultFail, + }, + { // 23 + fromDomain: "sub.example.org", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "mx.example.org", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultPass, + }, + } + for i, case_ := range cases { + test(i, case_) + } +} + +func TestExtractDomains(t *testing.T) { + type tCase struct { + hdr string + + fromDomain string + } + test := func(i int, c tCase) { + hdr, err := textproto.ReadHeader(bufio.NewReader(strings.NewReader(c.hdr + "\n\n"))) + if err != nil { + panic(err) + } + + domain, err := ExtractFromDomain(hdr) + if c.fromDomain == "" && err == nil { + t.Errorf("%d: expected failure, got fromDomain = %s", i, domain) + return + } + if c.fromDomain != "" && err != nil { + t.Errorf("%d: unexpected error: %v", i, err) + return + } + if domain != c.fromDomain { + t.Errorf("%d: want fromDomain = %v but got %s", i, c.fromDomain, domain) + } + } + + cases := []tCase{ + { + hdr: `From: `, + fromDomain: "example.org", + }, + { + hdr: `From: `, + fromDomain: "foo.example.org", + }, + { + hdr: `From: , `, + }, + { + hdr: `From: , +From: `, + }, + { + hdr: `From: `, + }, + { + hdr: `From: `, + }, + { + hdr: `From: foo`, + }, + } + for i, case_ := range cases { + test(i, case_) + } +} diff --git a/internal/dmarc/verifier.go b/internal/dmarc/verifier.go new file mode 100644 index 0000000..a79b526 --- /dev/null +++ b/internal/dmarc/verifier.go @@ -0,0 +1,174 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dmarc + +import ( + "context" + "math/rand" + "net" + "runtime/trace" + "strings" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/emersion/go-msgauth/dmarc" +) + +type verifyData struct { + policyDomain string + fromDomain string + record *Record + recordErr error +} + +// errPanic is used to propagate the panic() from the FetchRecord +// goroutine to the goroutine that called Apply. +type errPanic struct { + err interface{} +} + +func (errPanic) Error() string { + return "panic during policy fetch" +} + +// Verifier is the structure that wraps all state necessary to verify a +// single message using DMARC checks. +// +// It cannot be reused. +type Verifier struct { + fetchCh chan verifyData + fetchCancel context.CancelFunc + + resolver Resolver + + // TODO(GH #206): DMARC reporting + // FailureReportFunc is the callback that is called when a failure report + // is generated. If it is nil - failure reports generation is disabled. + // FailureReportFunc func(textproto.Header, io.Reader) +} + +func NewVerifier(r Resolver) *Verifier { + return &Verifier{ + fetchCh: make(chan verifyData, 1), + resolver: r, + } +} + +func (v *Verifier) Close() error { + if v.fetchCancel != nil { + v.fetchCancel() + } + return nil +} + +// FetchRecord prepares the Verifier by starting the policy lookup. Lookup is +// performed asynchronously to improve performance. +// +// If panic occurs in the lookup goroutine - call to Apply will panic. +func (v *Verifier) FetchRecord(ctx context.Context, header textproto.Header) { + fromDomain, err := ExtractFromDomain(header) + if err != nil { + v.fetchCh <- verifyData{ + recordErr: err, + } + return + } + + ctx, v.fetchCancel = context.WithCancel(ctx) + go func() { + defer func() { + if err := recover(); err != nil { + v.fetchCh <- verifyData{ + recordErr: errPanic{err: err}, + } + } + }() + + defer trace.StartRegion(ctx, "DMARC/FetchRecord").End() + + policyDomain, record, err := FetchRecord(ctx, v.resolver, fromDomain) + v.fetchCh <- verifyData{ + policyDomain: policyDomain, + fromDomain: fromDomain, + record: record, + recordErr: err, + } + }() +} + +// Apply actually performs all actions necessary to apply a DMARC policy to the message. +// +// The authRes slice should contain results for DKIM and SPF checks. FetchRecord should be +// caled before calling this function. +// +// It returns the Authentication-Result field to be included in the message (as +// a part of the EvalResult struct) and the appropriate action that should be +// taken by the MTA. In case of PolicyReject, caller should inspect the +// Result.Value to determine whether to use a temporary or permanent error code +// as Apply implements the 'fail closed' strategy for handling of temporary +// errors. +// +// Additionally, it relies on the math/rand default source to be initialized to determine +// whether to apply a policy with the pct key. +func (v *Verifier) Apply(authRes []authres.Result) (EvalResult, Policy) { + data := <-v.fetchCh + if data.recordErr != nil { + result := authres.DMARCResult{ + Value: authres.ResultPermError, + Reason: "Policy lookup failed: " + data.recordErr.Error(), + // If may be empty, but it is fine (it will not be included in the field then). + From: data.fromDomain, + } + if dnsErr, ok := data.recordErr.(*net.DNSError); ok && dnsErr.Temporary() { + result.Value = authres.ResultTempError + // 'fail closed' behavior, reject the message if a temporary error + // occurs. + return EvalResult{ + Authres: result, + }, dmarc.PolicyReject + } + return EvalResult{ + Authres: result, + }, dmarc.PolicyNone + } + if data.record == nil { + return EvalResult{ + Authres: authres.DMARCResult{ + Value: authres.ResultNone, + From: data.fromDomain, + }, + }, dmarc.PolicyNone + } + + result := EvaluateAlignment(data.fromDomain, data.record, authRes) + if result.Authres.Value == authres.ResultPass || result.Authres.Value == authres.ResultNone { + return result, dmarc.PolicyNone + } + + if data.record.Percent != nil && rand.Int31n(100) > int32(*data.record.Percent) { + return result, dmarc.PolicyNone + } + + policy := data.record.Policy + if !strings.EqualFold(data.policyDomain, data.fromDomain) && data.record.SubdomainPolicy != "" { + policy = data.record.SubdomainPolicy + } + + return result, policy +} diff --git a/internal/dmarc/verifier_test.go b/internal/dmarc/verifier_test.go new file mode 100644 index 0000000..1adfe99 --- /dev/null +++ b/internal/dmarc/verifier_test.go @@ -0,0 +1,219 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dmarc + +import ( + "bufio" + "context" + "errors" + "net" + "strings" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/foxcpp/go-mockdns" +) + +func TestDMARC(t *testing.T) { + test := func(zones map[string]mockdns.Zone, hdr string, authres []authres.Result, policyApplied Policy, dmarcRes authres.ResultValue) { + t.Helper() + v := NewVerifier(&mockdns.Resolver{Zones: zones}) + defer v.Close() + + hdrParsed, err := textproto.ReadHeader(bufio.NewReader(strings.NewReader(hdr))) + if err != nil { + panic(err) + } + v.FetchRecord(context.Background(), hdrParsed) + evalRes, policy := v.Apply(authres) + + if policy != policyApplied { + t.Errorf("expected applied policy to be '%v', got '%v'", policyApplied, policy) + } + if evalRes.Authres.Value != dmarcRes { + t.Errorf("expected DMARC result to be '%v', got '%v'", dmarcRes, evalRes.Authres.Value) + } + } + + // No policy => DMARC 'none' + test(map[string]mockdns.Zone{}, "From: hello@example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultNone) + + // Policy present & identifiers align => DMARC 'pass' + test(map[string]mockdns.Zone{ + "_dmarc.example.org.": { + TXT: []string{"v=DMARC1; p=none"}, + }, + }, "From: hello@example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultPass) + + // No SPF check run => DMARC 'none', no action taken + test(map[string]mockdns.Zone{ + "_dmarc.example.org.": { + TXT: []string{"v=DMARC1; p=reject"}, + }, + }, "From: hello@example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + }, PolicyNone, authres.ResultNone) + + // No DKIM check run => DMARC 'none', no action taken + test(map[string]mockdns.Zone{ + "_dmarc.example.org.": { + TXT: []string{"v=DMARC1; p=reject"}, + }, + }, "From: hello@example.org\r\n\r\n", []authres.Result{ + &authres.SPFResult{Value: authres.ResultPass, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultNone) + + // Check org. domain and from domain, prefer from domain. + // https://tools.ietf.org/html/rfc7489#section-6.6.3 + test(map[string]mockdns.Zone{ + "_dmarc.example.org.": { + TXT: []string{"v=DMARC1; p=none"}, + }, + }, "From: hello@sub.example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultPass) + test(map[string]mockdns.Zone{ + "_dmarc.sub.example.org.": { + TXT: []string{"v=DMARC1; p=none"}, + }, + }, "From: hello@sub.example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultPass) + test(map[string]mockdns.Zone{ + "_dmarc.sub.example.org.": { + TXT: []string{"v=DMARC1; p=none"}, + }, + "_dmarc.example.org.": { + TXT: []string{"v=malformed"}, + }, + }, "From: hello@sub.example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultPass) + + // Non-DMARC records are ignored. + // https://tools.ietf.org/html/rfc7489#section-6.6.3 + test(map[string]mockdns.Zone{ + "_dmarc.example.org.": { + TXT: []string{"ignore", "v=DMARC1; p=none"}, + }, + }, "From: hello@sub.example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultPass) + + // Multiple policies => no policy. + // https://tools.ietf.org/html/rfc7489#section-6.6.3 + test(map[string]mockdns.Zone{ + "_dmarc.example.org.": { + TXT: []string{"v=DMARC1; p=reject", "v=DMARC1; p=none"}, + }, + }, "From: hello@sub.example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultNone) + + // Malformed policy => no policy + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + TXT: []string{"v=aaaa"}, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultNone) + + // Policy fetch error => DMARC 'permerror' but the message + // is accepted. + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + Err: errors.New("the dns server is going insane"), + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultPermError) + + // Policy fetch error => DMARC 'temperror' but the message + // is accepted ("fail closed") + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + Err: &net.DNSError{ + Err: "the dns server is going insane, temporary", + IsTemporary: true, + }, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyReject, authres.ResultTempError) + + // Misaligned From vs DKIM => DMARC 'fail'. + // Side note: More comprehensive tests for alignment evaluation + // can be found in check/dmarc/evaluate_test.go. This test merely checks + // that the correct action is taken based on the policy. + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + TXT: []string{"v=DMARC1; p=none"}, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultFail) + + // Misaligned From vs DKIM => DMARC 'fail', policy says to reject + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + TXT: []string{"v=DMARC1; p=reject"}, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyReject, authres.ResultFail) + + // Misaligned From vs DKIM => DMARC 'fail' + // Subdomain policy requests no action, main domain policy says to reject. + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + TXT: []string{"v=DMARC1; sp=none; p=reject"}, + }, + }, "From: hello@sub.example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultFail) + + // Misaligned From vs DKIM => DMARC 'fail', policy says to quarantine. + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + TXT: []string{"v=DMARC1; p=quarantine"}, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyQuarantine, authres.ResultFail) +} diff --git a/internal/dsn/dsn.go b/internal/dsn/dsn.go new file mode 100644 index 0000000..59707a7 --- /dev/null +++ b/internal/dsn/dsn.go @@ -0,0 +1,298 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package dsn contains the utilities used for dsn message (DSN) generation. +// +// It implements RFC 3464 and RFC 3462. +package dsn + +import ( + "errors" + "fmt" + "io" + "strings" + "text/template" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/dns" +) + +type ReportingMTAInfo struct { + ReportingMTA string + ReceivedFromMTA string + + // Message sender address, included as 'X-Maddy-Sender: rfc822; ADDR' field. + XSender string + + // Message identifier, included as 'X-Maddy-MsgId: MSGID' field. + XMessageID string + + // Time when message was enqueued for delivery by Reporting MTA. + ArrivalDate time.Time + + // Time when message delivery was attempted last time. + LastAttemptDate time.Time +} + +func (info ReportingMTAInfo) WriteTo(utf8 bool, w io.Writer) error { + // DSN format uses structure similar to MIME header, so we reuse + // MIME generator here. + h := textproto.Header{} + + if info.ReportingMTA == "" { + return errors.New("dsn: Reporting-MTA field is mandatory") + } + + reportingMTA, err := dns.SelectIDNA(utf8, info.ReportingMTA) + if err != nil { + return fmt.Errorf("dsn: cannot convert Reporting-MTA to a suitable representation: %w", err) + } + + h.Add("Reporting-MTA", "dns; "+reportingMTA) + + if info.ReceivedFromMTA != "" { + receivedFromMTA, err := dns.SelectIDNA(utf8, info.ReceivedFromMTA) + if err != nil { + return fmt.Errorf("dsn: cannot convert Received-From-MTA to a suitable representation: %w", err) + } + + h.Add("Received-From-MTA", "dns; "+receivedFromMTA) + } + + if info.XSender != "" { + sender, err := address.SelectIDNA(utf8, info.XSender) + if err != nil { + return fmt.Errorf("dsn: cannot convert X-Maddy-Sender to a suitable representation: %w", err) + } + + if utf8 { + h.Add("X-Maddy-Sender", "utf8; "+sender) + } else { + h.Add("X-Maddy-Sender", "rfc822; "+sender) + } + } + if info.XMessageID != "" { + h.Add("X-Maddy-MsgID", info.XMessageID) + } + + if !info.ArrivalDate.IsZero() { + h.Add("Arrival-Date", info.ArrivalDate.Format("Mon, 2 Jan 2006 15:04:05 -0700")) + } + if !info.ArrivalDate.IsZero() { + h.Add("Last-Attempt-Date", info.LastAttemptDate.Format("Mon, 2 Jan 2006 15:04:05 -0700")) + } + + return textproto.WriteHeader(w, h) +} + +type Action string + +const ( + ActionFailed Action = "failed" + ActionDelayed Action = "delayed" + ActionDelivered Action = "delivered" + ActionRelayed Action = "relayed" + ActionExpanded Action = "expanded" +) + +type RecipientInfo struct { + FinalRecipient string + RemoteMTA string + + Action Action + Status smtp.EnhancedCode + + // DiagnosticCode is the error that will be returned to the sender. + DiagnosticCode error +} + +func (info RecipientInfo) WriteTo(utf8 bool, w io.Writer) error { + // DSN format uses structure similar to MIME header, so we reuse + // MIME generator here. + h := textproto.Header{} + + if info.FinalRecipient == "" { + return errors.New("dsn: Final-Recipient is required") + } + finalRcpt, err := address.SelectIDNA(utf8, info.FinalRecipient) + if err != nil { + return fmt.Errorf("dsn: cannot convert Final-Recipient to a suitable representation: %w", err) + } + if utf8 { + h.Add("Final-Recipient", "utf8; "+finalRcpt) + } else { + h.Add("Final-Recipient", "rfc822; "+finalRcpt) + } + + if info.Action == "" { + return errors.New("dsn: Action is required") + } + h.Add("Action", string(info.Action)) + if info.Status[0] == 0 { + return errors.New("dsn: Status is required") + } + h.Add("Status", fmt.Sprintf("%d.%d.%d", info.Status[0], info.Status[1], info.Status[2])) + + if smtpErr, ok := info.DiagnosticCode.(*smtp.SMTPError); ok { + // Error message may contain newlines if it is received from another SMTP server. + // But we cannot directly insert CR/LF into Disagnostic-Code so rewrite it. + h.Add("Diagnostic-Code", fmt.Sprintf("smtp; %d %d.%d.%d %s", + smtpErr.Code, smtpErr.EnhancedCode[0], smtpErr.EnhancedCode[1], smtpErr.EnhancedCode[2], + strings.ReplaceAll(strings.ReplaceAll(smtpErr.Message, "\n", " "), "\r", " "))) + } else if utf8 { + // It might contain Unicode, so don't include it if we are not allowed to. + // ... I didn't bother implementing mangling logic to remove Unicode + // characters. + errorDesc := info.DiagnosticCode.Error() + errorDesc = strings.ReplaceAll(strings.ReplaceAll(errorDesc, "\n", " "), "\r", " ") + + h.Add("Diagnostic-Code", "X-Maddy; "+errorDesc) + } + + if info.RemoteMTA != "" { + remoteMTA, err := dns.SelectIDNA(utf8, info.RemoteMTA) + if err != nil { + return fmt.Errorf("dsn: cannot convert Remote-MTA to a suitable representation: %w", err) + } + + h.Add("Remote-MTA", "dns; "+remoteMTA) + } + + return textproto.WriteHeader(w, h) +} + +type Envelope struct { + MsgID string + From string + To string +} + +// GenerateDSN is a top-level function that should be used for generation of the DSNs. +// +// DSN header will be returned, body itself will be written to outWriter. +func GenerateDSN(utf8 bool, envelope Envelope, mtaInfo ReportingMTAInfo, rcptsInfo []RecipientInfo, failedHeader textproto.Header, outWriter io.Writer) (textproto.Header, error) { + partWriter := textproto.NewMultipartWriter(outWriter) + + reportHeader := textproto.Header{} + reportHeader.Add("Date", time.Now().Format("Mon, 2 Jan 2006 15:04:05 -0700")) + reportHeader.Add("Message-Id", envelope.MsgID) + reportHeader.Add("Content-Transfer-Encoding", "8bit") + reportHeader.Add("Content-Type", "multipart/report; report-type=delivery-status; boundary="+partWriter.Boundary()) + reportHeader.Add("MIME-Version", "1.0") + reportHeader.Add("Auto-Submitted", "auto-replied") + reportHeader.Add("To", envelope.To) + reportHeader.Add("From", envelope.From) + reportHeader.Add("Subject", "Undelivered Mail Returned to Sender") + + defer partWriter.Close() + + if err := writeHumanReadablePart(partWriter, mtaInfo, rcptsInfo); err != nil { + return textproto.Header{}, err + } + if err := writeMachineReadablePart(utf8, partWriter, mtaInfo, rcptsInfo); err != nil { + return textproto.Header{}, err + } + return reportHeader, writeHeader(utf8, partWriter, failedHeader) +} + +func writeHeader(utf8 bool, w *textproto.MultipartWriter, header textproto.Header) error { + partHeader := textproto.Header{} + partHeader.Add("Content-Description", "Undelivered message header") + if utf8 { + partHeader.Add("Content-Type", "message/global-headers") + } else { + partHeader.Add("Content-Type", "message/rfc822-headers") + } + partHeader.Add("Content-Transfer-Encoding", "8bit") + headerWriter, err := w.CreatePart(partHeader) + if err != nil { + return err + } + return textproto.WriteHeader(headerWriter, header) +} + +func writeMachineReadablePart(utf8 bool, w *textproto.MultipartWriter, mtaInfo ReportingMTAInfo, rcptsInfo []RecipientInfo) error { + machineHeader := textproto.Header{} + if utf8 { + machineHeader.Add("Content-Type", "message/global-delivery-status") + } else { + machineHeader.Add("Content-Type", "message/delivery-status") + } + machineHeader.Add("Content-Description", "Delivery report") + machineWriter, err := w.CreatePart(machineHeader) + if err != nil { + return err + } + + // WriteTo will add an empty line after output. + if err := mtaInfo.WriteTo(utf8, machineWriter); err != nil { + return err + } + + for _, rcpt := range rcptsInfo { + if err := rcpt.WriteTo(utf8, machineWriter); err != nil { + return err + } + } + return nil +} + +// failedText is the text of the human-readable part of DSN. +var failedText = template.Must(template.New("dsn-text").Parse(` +This is the mail delivery system at {{.ReportingMTA}}. + +Unfortunately, your message could not be delivered to one or more +recipients. The usual cause of this problem is invalid +recipient address or maintenance at the recipient side. + +Contact the postmaster for further assistance, provide the Message ID (below): + +Message ID: {{.XMessageID}} +Arrival: {{.ArrivalDate}} +Last delivery attempt: {{.LastAttemptDate}} + +`)) + +func writeHumanReadablePart(w *textproto.MultipartWriter, mtaInfo ReportingMTAInfo, rcptsInfo []RecipientInfo) error { + humanHeader := textproto.Header{} + humanHeader.Add("Content-Transfer-Encoding", "8bit") + humanHeader.Add("Content-Type", `text/plain; charset="utf-8"`) + humanHeader.Add("Content-Description", "Notification") + humanWriter, err := w.CreatePart(humanHeader) + if err != nil { + return err + } + + mtaInfo.ArrivalDate = mtaInfo.ArrivalDate.Truncate(time.Second) + mtaInfo.LastAttemptDate = mtaInfo.LastAttemptDate.Truncate(time.Second) + + if err := failedText.Execute(humanWriter, mtaInfo); err != nil { + return err + } + + for _, rcpt := range rcptsInfo { + if _, err := fmt.Fprintf(humanWriter, "Delivery to %s failed with error: %v\n", rcpt.FinalRecipient, rcpt.DiagnosticCode); err != nil { + return err + } + } + + return nil +} diff --git a/internal/endpoint/dovecot_sasld/dovecot_sasl.go b/internal/endpoint/dovecot_sasld/dovecot_sasl.go new file mode 100644 index 0000000..2679696 --- /dev/null +++ b/internal/endpoint/dovecot_sasld/dovecot_sasl.go @@ -0,0 +1,127 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dovecotsasld + +import ( + "fmt" + stdlog "log" + "net" + "strings" + "sync" + + "github.com/emersion/go-sasl" + dovecotsasl "github.com/foxcpp/go-dovecot-sasl" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth" + "github.com/foxcpp/maddy/internal/authz" +) + +const modName = "dovecot_sasld" + +type Endpoint struct { + addrs []string + log log.Logger + saslAuth auth.SASLAuth + + listenersWg sync.WaitGroup + + srv *dovecotsasl.Server +} + +func New(_ string, addrs []string) (module.Module, error) { + return &Endpoint{ + addrs: addrs, + saslAuth: auth.SASLAuth{ + Log: log.Logger{Name: modName + "/saslauth"}, + }, + log: log.Logger{Name: modName, Debug: log.DefaultLogger.Debug}, + }, nil +} + +func (endp *Endpoint) Name() string { + return modName +} + +func (endp *Endpoint) InstanceName() string { + return modName +} + +func (endp *Endpoint) Init(cfg *config.Map) error { + cfg.Callback("auth", func(m *config.Map, node config.Node) error { + return endp.saslAuth.AddProvider(m, node) + }) + cfg.Bool("sasl_login", false, false, &endp.saslAuth.EnableLogin) + config.EnumMapped(cfg, "auth_map_normalize", true, false, authz.NormalizeFuncs, authz.NormalizeAuto, + &endp.saslAuth.AuthNormalize) + modconfig.Table(cfg, "auth_map", true, false, nil, &endp.saslAuth.AuthMap) + if _, err := cfg.Process(); err != nil { + return err + } + + endp.srv = dovecotsasl.NewServer() + endp.srv.Log = stdlog.New(endp.log, "", 0) + endp.saslAuth.Log.Debug = endp.log.Debug + + for _, mech := range endp.saslAuth.SASLMechanisms() { + endp.srv.AddMechanism(mech, mechInfo[mech], func(req *dovecotsasl.AuthReq) sasl.Server { + var remoteAddr net.Addr + if req.RemoteIP != nil && req.RemotePort != 0 { + remoteAddr = &net.TCPAddr{IP: req.RemoteIP, Port: int(req.RemotePort)} + } + + return endp.saslAuth.CreateSASL(mech, remoteAddr, func(_ string, _ auth.ContextData) error { return nil }) + }) + } + + for _, addr := range endp.addrs { + parsed, err := config.ParseEndpoint(addr) + if err != nil { + return fmt.Errorf("%s: %v", modName, err) + } + + l, err := net.Listen(parsed.Network(), parsed.Address()) + if err != nil { + return fmt.Errorf("%s: %v", modName, err) + } + endp.log.Printf("listening on %v", l.Addr()) + + endp.listenersWg.Add(1) + go func() { + defer endp.listenersWg.Done() + if err := endp.srv.Serve(l); err != nil { + if !strings.HasSuffix(err.Error(), "use of closed network connection") { + endp.log.Printf("failed to serve %v: %v", l.Addr(), err) + } + } + }() + } + + return nil +} + +func (endp *Endpoint) Close() error { + return endp.srv.Close() +} + +func init() { + module.RegisterEndpoint(modName, New) +} diff --git a/internal/endpoint/dovecot_sasld/mech_info.go b/internal/endpoint/dovecot_sasld/mech_info.go new file mode 100644 index 0000000..ab7b5f8 --- /dev/null +++ b/internal/endpoint/dovecot_sasld/mech_info.go @@ -0,0 +1,33 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dovecotsasld + +import ( + "github.com/emersion/go-sasl" + dovecotsasl "github.com/foxcpp/go-dovecot-sasl" +) + +var mechInfo = map[string]dovecotsasl.Mechanism{ + sasl.Plain: { + Plaintext: true, + }, + sasl.Login: { + Plaintext: true, + }, +} diff --git a/internal/endpoint/imap/imap.go b/internal/endpoint/imap/imap.go new file mode 100644 index 0000000..cece797 --- /dev/null +++ b/internal/endpoint/imap/imap.go @@ -0,0 +1,322 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imap + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "strings" + "sync" + + "github.com/emersion/go-imap" + compress "github.com/emersion/go-imap-compress" + sortthread "github.com/emersion/go-imap-sortthread" + imapbackend "github.com/emersion/go-imap/backend" + imapserver "github.com/emersion/go-imap/server" + "github.com/emersion/go-message" + _ "github.com/emersion/go-message/charset" + "github.com/emersion/go-sasl" + i18nlevel "github.com/foxcpp/go-imap-i18nlevel" + namespace "github.com/foxcpp/go-imap-namespace" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + tls2 "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth" + "github.com/foxcpp/maddy/internal/authz" + "github.com/foxcpp/maddy/internal/proxy_protocol" + "github.com/foxcpp/maddy/internal/updatepipe" +) + +type Endpoint struct { + addrs []string + serv *imapserver.Server + listeners []net.Listener + proxyProtocol *proxy_protocol.ProxyProtocol + Store module.Storage + + tlsConfig *tls.Config + listenersWg sync.WaitGroup + + saslAuth auth.SASLAuth + + storageNormalize authz.NormalizeFunc + storageMap module.Table + + Log log.Logger +} + +func New(modName string, addrs []string) (module.Module, error) { + endp := &Endpoint{ + addrs: addrs, + Log: log.Logger{Name: modName}, + saslAuth: auth.SASLAuth{ + Log: log.Logger{Name: modName + "/sasl"}, + }, + } + + return endp, nil +} + +func (endp *Endpoint) Init(cfg *config.Map) error { + var ( + insecureAuth bool + ioDebug bool + ioErrors bool + ) + + cfg.Callback("auth", func(m *config.Map, node config.Node) error { + return endp.saslAuth.AddProvider(m, node) + }) + cfg.Bool("sasl_login", false, false, &endp.saslAuth.EnableLogin) + cfg.Custom("storage", false, true, nil, modconfig.StorageDirective, &endp.Store) + cfg.Custom("tls", true, true, nil, tls2.TLSDirective, &endp.tlsConfig) + cfg.Custom("proxy_protocol", false, false, nil, proxy_protocol.ProxyProtocolDirective, &endp.proxyProtocol) + cfg.Bool("insecure_auth", false, false, &insecureAuth) + cfg.Bool("io_debug", false, false, &ioDebug) + cfg.Bool("io_errors", false, false, &ioErrors) + cfg.Bool("debug", true, false, &endp.Log.Debug) + config.EnumMapped(cfg, "storage_map_normalize", false, false, authz.NormalizeFuncs, authz.NormalizeAuto, + &endp.storageNormalize) + modconfig.Table(cfg, "storage_map", false, false, nil, &endp.storageMap) + config.EnumMapped(cfg, "auth_map_normalize", true, false, authz.NormalizeFuncs, authz.NormalizeAuto, + &endp.saslAuth.AuthNormalize) + modconfig.Table(cfg, "auth_map", true, false, nil, &endp.saslAuth.AuthMap) + if _, err := cfg.Process(); err != nil { + return err + } + + if updBe, ok := endp.Store.(updatepipe.Backend); ok { + if err := updBe.EnableUpdatePipe(updatepipe.ModeReplicate); err != nil { + endp.Log.Error("failed to initialize updates pipe", err) + } + } + + endp.saslAuth.Log.Debug = endp.Log.Debug + + addresses := make([]config.Endpoint, 0, len(endp.addrs)) + for _, addr := range endp.addrs { + saddr, err := config.ParseEndpoint(addr) + if err != nil { + return fmt.Errorf("imap: invalid address: %s", addr) + } + addresses = append(addresses, saddr) + } + + endp.serv = imapserver.New(endp) + endp.serv.AllowInsecureAuth = insecureAuth + endp.serv.TLSConfig = endp.tlsConfig + if ioErrors { + endp.serv.ErrorLog = &endp.Log + } else { + endp.serv.ErrorLog = log.Logger{Out: log.NopOutput{}} + } + if ioDebug { + endp.serv.Debug = endp.Log.DebugWriter() + endp.Log.Println("I/O debugging is on! It may leak passwords in logs, be careful!") + } + + if err := endp.enableExtensions(); err != nil { + return err + } + + for _, mech := range endp.saslAuth.SASLMechanisms() { + endp.serv.EnableAuth(mech, func(c imapserver.Conn) sasl.Server { + return endp.saslAuth.CreateSASL(mech, c.Info().RemoteAddr, func(identity string, data auth.ContextData) error { + return endp.openAccount(c, identity) + }) + }) + } + + return endp.setupListeners(addresses) +} + +func (endp *Endpoint) setupListeners(addresses []config.Endpoint) error { + for _, addr := range addresses { + var l net.Listener + var err error + l, err = net.Listen(addr.Network(), addr.Address()) + if err != nil { + return fmt.Errorf("imap: %v", err) + } + endp.Log.Printf("listening on %v", addr) + + if addr.IsTLS() { + if endp.tlsConfig == nil { + return errors.New("imap: can't bind on IMAPS endpoint without TLS configuration") + } + l = tls.NewListener(l, endp.tlsConfig) + } + + if endp.proxyProtocol != nil { + l = proxy_protocol.NewListener(l, endp.proxyProtocol, endp.Log) + } + + endp.listeners = append(endp.listeners, l) + + endp.listenersWg.Add(1) + go func() { + if err := endp.serv.Serve(l); err != nil && !strings.HasSuffix(err.Error(), "use of closed network connection") { + endp.Log.Printf("imap: failed to serve %s: %s", addr, err) + } + endp.listenersWg.Done() + }() + } + + if endp.serv.AllowInsecureAuth { + endp.Log.Println("authentication over unencrypted connections is allowed, this is insecure configuration and should be used only for testing!") + } + if endp.serv.TLSConfig == nil { + endp.Log.Println("TLS is disabled, this is insecure configuration and should be used only for testing!") + endp.serv.AllowInsecureAuth = true + } + + return nil +} + +func (endp *Endpoint) Name() string { + return "imap" +} + +func (endp *Endpoint) InstanceName() string { + return "imap" +} + +func (endp *Endpoint) Close() error { + for _, l := range endp.listeners { + l.Close() + } + if err := endp.serv.Close(); err != nil { + return err + } + endp.listenersWg.Wait() + return nil +} + +func (endp *Endpoint) usernameForStorage(ctx context.Context, saslUsername string) (string, error) { + saslUsername, err := endp.storageNormalize(saslUsername) + if err != nil { + return "", err + } + + if endp.storageMap == nil { + return saslUsername, nil + } + + mapped, ok, err := endp.storageMap.Lookup(ctx, saslUsername) + if err != nil { + return "", err + } + if !ok { + return "", imapbackend.ErrInvalidCredentials + } + + if saslUsername != mapped { + endp.Log.DebugMsg("using mapped username for storage", "username", saslUsername, "mapped_username", mapped) + } + + return mapped, nil +} + +func (endp *Endpoint) openAccount(c imapserver.Conn, identity string) error { + username, err := endp.usernameForStorage(context.TODO(), identity) + if err != nil { + if errors.Is(err, imapbackend.ErrInvalidCredentials) { + return err + } + endp.Log.Error("failed to determine storage account name", err, "username", username) + return fmt.Errorf("internal server error") + } + + u, err := endp.Store.GetOrCreateIMAPAcct(username) + if err != nil { + return err + } + ctx := c.Context() + ctx.State = imap.AuthenticatedState + ctx.User = u + return nil +} + +func (endp *Endpoint) Login(connInfo *imap.ConnInfo, username, password string) (imapbackend.User, error) { + // saslAuth handles AuthMap calling. + err := endp.saslAuth.AuthPlain(username, password) + if err != nil { + endp.Log.Error("authentication failed", err, "username", username, "src_ip", connInfo.RemoteAddr) + return nil, imapbackend.ErrInvalidCredentials + } + + storageUsername, err := endp.usernameForStorage(context.TODO(), username) + if err != nil { + if errors.Is(err, imapbackend.ErrInvalidCredentials) { + return nil, err + } + endp.Log.Error("authentication failed due to an internal error", err, "username", username, "src_ip", connInfo.RemoteAddr) + return nil, fmt.Errorf("internal server error") + } + + return endp.Store.GetOrCreateIMAPAcct(storageUsername) +} + +func (endp *Endpoint) I18NLevel() int { + be, ok := endp.Store.(i18nlevel.Backend) + if !ok { + return 0 + } + return be.I18NLevel() +} + +func (endp *Endpoint) enableExtensions() error { + exts := endp.Store.IMAPExtensions() + for _, ext := range exts { + switch ext { + case "I18NLEVEL=1", "I18NLEVEL=2": + endp.serv.Enable(i18nlevel.NewExtension()) + case "SORT": + endp.serv.Enable(sortthread.NewSortExtension()) + } + if strings.HasPrefix(ext, "THREAD") { + endp.serv.Enable(sortthread.NewThreadExtension()) + } + } + + endp.serv.Enable(compress.NewExtension()) + endp.serv.Enable(namespace.NewExtension()) + + return nil +} + +func (endp *Endpoint) SupportedThreadAlgorithms() []sortthread.ThreadAlgorithm { + be, ok := endp.Store.(sortthread.ThreadBackend) + if !ok { + return nil + } + + return be.SupportedThreadAlgorithms() +} + +func init() { + module.RegisterEndpoint("imap", New) + + imap.CharsetReader = message.CharsetReader +} diff --git a/internal/endpoint/openmetrics/om.go b/internal/endpoint/openmetrics/om.go new file mode 100644 index 0000000..874a333 --- /dev/null +++ b/internal/endpoint/openmetrics/om.go @@ -0,0 +1,107 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package openmetrics + +import ( + "errors" + "fmt" + "net" + "net/http" + "sync" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +const modName = "openmetrics" + +type Endpoint struct { + addrs []string + logger log.Logger + + listenersWg sync.WaitGroup + serv http.Server + mux *http.ServeMux +} + +func New(_ string, args []string) (module.Module, error) { + return &Endpoint{ + addrs: args, + logger: log.Logger{Name: modName, Debug: log.DefaultLogger.Debug}, + }, nil +} + +func (e *Endpoint) Init(cfg *config.Map) error { + cfg.Bool("debug", false, false, &e.logger.Debug) + if _, err := cfg.Process(); err != nil { + return err + } + + e.mux = http.NewServeMux() + e.mux.Handle("/metrics", promhttp.Handler()) + e.serv.Handler = e.mux + + for _, a := range e.addrs { + endp, err := config.ParseEndpoint(a) + if err != nil { + return fmt.Errorf("%s: malformed endpoint: %v", modName, err) + } + if endp.IsTLS() { + return fmt.Errorf("%s: TLS is not supported yet", modName) + } + l, err := net.Listen(endp.Network(), endp.Address()) + if err != nil { + return fmt.Errorf("%s: %v", modName, err) + } + + e.listenersWg.Add(1) + go func() { + e.logger.Println("listening on", endp.String()) + err := e.serv.Serve(l) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + e.logger.Error("serve failed", err, "endpoint", a) + } + e.listenersWg.Done() + }() + } + + return nil +} + +func (e *Endpoint) Name() string { + return modName +} + +func (e *Endpoint) InstanceName() string { + return "" +} + +func (e *Endpoint) Close() error { + if err := e.serv.Close(); err != nil { + return err + } + e.listenersWg.Wait() + return nil +} + +func init() { + module.RegisterEndpoint(modName, New) +} diff --git a/internal/endpoint/smtp/date.go b/internal/endpoint/smtp/date.go new file mode 100644 index 0000000..e41daa1 --- /dev/null +++ b/internal/endpoint/smtp/date.go @@ -0,0 +1,66 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import ( + "fmt" + "regexp" + "time" +) + +// Taken from https://github.com/emersion/go-imap/blob/09c1d69/date.go. + +var dateTimeLayouts = [...]string{ + // Defined in RFC 5322 section 3.3, mentioned as env-date in RFC 3501 page 84. + "Mon, 02 Jan 2006 15:04:05 -0700", + "_2 Jan 2006 15:04:05 -0700", + "_2 Jan 2006 15:04:05 MST", + "_2 Jan 2006 15:04 -0700", + "_2 Jan 2006 15:04 MST", + "_2 Jan 06 15:04:05 -0700", + "_2 Jan 06 15:04:05 MST", + "_2 Jan 06 15:04 -0700", + "_2 Jan 06 15:04 MST", + "Mon, _2 Jan 2006 15:04:05 -0700", + "Mon, _2 Jan 2006 15:04:05 MST", + "Mon, _2 Jan 2006 15:04 -0700", + "Mon, _2 Jan 2006 15:04 MST", + "Mon, _2 Jan 06 15:04:05 -0700", + "Mon, _2 Jan 06 15:04:05 MST", + "Mon, _2 Jan 06 15:04 -0700", + "Mon, _2 Jan 06 15:04 MST", +} + +// TODO: this is a blunt way to strip any trailing CFWS (comment). A sharper +// one would strip multiple CFWS, and only if really valid according to +// RFC5322. +var commentRE = regexp.MustCompile(`[ \t]+\(.*\)$`) + +// Try parsing the date based on the layouts defined in RFC 5322, section 3.3. +// Inspired by https://github.com/golang/go/blob/master/src/net/mail/message.go +func parseMessageDateTime(maybeDate string) (time.Time, error) { + maybeDate = commentRE.ReplaceAllString(maybeDate, "") + for _, layout := range dateTimeLayouts { + parsed, err := time.Parse(layout, maybeDate) + if err == nil { + return parsed, nil + } + } + return time.Time{}, fmt.Errorf("date %s could not be parsed", maybeDate) +} diff --git a/internal/endpoint/smtp/metrics.go b/internal/endpoint/smtp/metrics.go new file mode 100644 index 0000000..509241b --- /dev/null +++ b/internal/endpoint/smtp/metrics.go @@ -0,0 +1,87 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import "github.com/prometheus/client_golang/prometheus" + +var ( + startedSMTPTransactions = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "smtp", + Name: "started_transactions", + Help: "Amount of SMTP trasanactions started", + }, + []string{"module"}, + ) + completedSMTPTransactions = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "smtp", + Name: "smtp_completed_transactions", + Help: "Amount of SMTP trasanactions successfully completed", + }, + []string{"module"}, + ) + abortedSMTPTransactions = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "smtp", + Name: "aborted_transactions", + Help: "Amount of SMTP trasanactions aborted", + }, + []string{"module"}, + ) + + ratelimitDefers = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "smtp", + Name: "ratelimit_deferred", + Help: "Messages rejected with 4xx code due to ratelimiting", + }, + []string{"module"}, + ) + failedLogins = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "smtp", + Name: "failed_logins", + Help: "AUTH command failures", + }, + []string{"module"}, + ) + failedCmds = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "smtp", + Name: "failed_commands", + Help: "Failed transaction commands (MAIL, RCPT, DATA)", + }, + []string{"module", "command", "smtp_code", "smtp_enchcode"}, + ) +) + +func init() { + prometheus.MustRegister(startedSMTPTransactions) + prometheus.MustRegister(completedSMTPTransactions) + prometheus.MustRegister(abortedSMTPTransactions) + prometheus.MustRegister(ratelimitDefers) + prometheus.MustRegister(failedCmds) +} diff --git a/internal/endpoint/smtp/session.go b/internal/endpoint/smtp/session.go new file mode 100644 index 0000000..cff1c01 --- /dev/null +++ b/internal/endpoint/smtp/session.go @@ -0,0 +1,656 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "net" + "runtime/trace" + "strconv" + "strings" + "sync" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-sasl" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth" +) + +func limitReader(r io.Reader, n int64, err error) *limitedReader { + return &limitedReader{R: r, N: n, E: err, Enabled: true} +} + +type limitedReader struct { + R io.Reader + N int64 + E error + Enabled bool +} + +// same as io.LimitedReader.Read except returning the custom error and the option +// to be disabled +func (l *limitedReader) Read(p []byte) (n int, err error) { + if !l.Enabled { + return l.R.Read(p) + } + if l.N <= 0 { + return 0, l.E + } + if int64(len(p)) > l.N { + p = p[0:l.N] + } + n, err = l.R.Read(p) + l.N -= int64(n) + return +} + +type Session struct { + endp *Endpoint + + // Specific for this session. + // sessionCtx is not used for cancellation or timeouts, only for tracing. + sessionCtx context.Context + cancelRDNS func() + connState module.ConnState + repeatedMailErrs int + loggedRcptErrors int + + // Specific for the currently handled message. + // msgCtx is not used for cancellation or timeouts, only for tracing. + // It is the subcontext of sessionCtx. + // Mutex is used to prevent Close from accessing inconsistent state when it + // is called asynchronously to any SMTP command. + msgLock sync.Mutex + msgCtx context.Context + msgTask *trace.Task + mailFrom string + opts smtp.MailOptions + msgMeta *module.MsgMetadata + delivery module.Delivery + deliveryErr error + + log log.Logger +} + +func (s *Session) AuthMechanisms() []string { + return s.endp.saslAuth.SASLMechanisms() +} + +func (s *Session) Auth(mech string) (sasl.Server, error) { + return s.endp.saslAuth.CreateSASL(mech, s.connState.RemoteAddr, func(identity string, data auth.ContextData) error { + s.connState.AuthUser = identity + s.connState.AuthPassword = data.Password + return nil + }), nil +} + +func (s *Session) Reset() { + s.msgLock.Lock() + defer s.msgLock.Unlock() + + if s.delivery != nil { + s.abort(s.msgCtx) + } + s.endp.Log.DebugMsg("reset") +} + +func (s *Session) releaseLimits() { + domain := "" + if s.mailFrom != "" { + var err error + _, domain, err = address.Split(s.mailFrom) + if err != nil { + return + } + } + + addr, ok := s.msgMeta.Conn.RemoteAddr.(*net.TCPAddr) + if !ok { + addr = &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)} + } + s.endp.limits.ReleaseMsg(addr.IP, domain) +} + +func (s *Session) abort(ctx context.Context) { + if err := s.delivery.Abort(ctx); err != nil { + s.endp.Log.Error("delivery abort failed", err) + } + s.log.Msg("aborted", "msg_id", s.msgMeta.ID) + abortedSMTPTransactions.WithLabelValues(s.endp.name).Inc() + s.cleanSession() +} + +func (s *Session) cleanSession() { + s.releaseLimits() + + s.mailFrom = "" + s.opts = smtp.MailOptions{} + s.msgMeta = nil + s.delivery = nil + s.deliveryErr = nil + s.msgCtx = nil + s.msgTask.End() +} + +func (s *Session) AuthPlain(username, password string) error { + // Executed before authentication and session initialization. + if err := s.endp.pipeline.RunEarlyChecks(context.TODO(), &s.connState); err != nil { + return s.endp.wrapErr("", true, "AUTH", err) + } + + // saslAuth will handle AuthMap and AuthNormalize. + err := s.endp.saslAuth.AuthPlain(username, password) + if err != nil { + s.endp.Log.Error("authentication failed", err, "username", username, "src_ip", s.connState.RemoteAddr) + + failedLogins.WithLabelValues(s.endp.name).Inc() + + if exterrors.IsTemporary(err) { + return &smtp.SMTPError{ + Code: 454, + EnhancedCode: smtp.EnhancedCode{4, 7, 0}, + Message: "Temporary authentication failure", + } + } + + return &smtp.SMTPError{ + Code: 535, + EnhancedCode: smtp.EnhancedCode{5, 7, 8}, + Message: "Invalid credentials", + } + } + + s.connState.AuthUser = username + s.connState.AuthPassword = password + + return nil +} + +func (s *Session) startDelivery(ctx context.Context, from string, opts smtp.MailOptions) (string, error) { + var err error + msgMeta := &module.MsgMetadata{ + Conn: &s.connState, + SMTPOpts: opts, + } + msgMeta.ID, err = module.GenerateMsgID() + if err != nil { + return "", err + } + + if s.connState.AuthUser != "" { + s.log.Msg("incoming message", + "src_host", msgMeta.Conn.Hostname, + "src_ip", msgMeta.Conn.RemoteAddr.String(), + "sender", from, + "msg_id", msgMeta.ID, + "username", s.connState.AuthUser, + ) + } else { + s.log.Msg("incoming message", + "src_host", msgMeta.Conn.Hostname, + "src_ip", msgMeta.Conn.RemoteAddr.String(), + "sender", from, + "msg_id", msgMeta.ID, + ) + } + + // INTERNATIONALIZATION: Do not permit non-ASCII addresses unless SMTPUTF8 is + // used. + if !opts.UTF8 { + for _, ch := range from { + if ch > 128 { + return "", &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 6, 7}, + Message: "SMTPUTF8 is required for non-ASCII senders", + } + } + } + } + + // Decode punycode, normalize to NFC and case-fold address. + cleanFrom := from + if from != "" { + cleanFrom, err = address.CleanDomain(from) + if err != nil { + return "", &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 1, 7}, + Message: "Unable to normalize the sender address", + } + } + } + + msgMeta.OriginalFrom = from + + domain := "" + if cleanFrom != "" { + _, domain, err = address.Split(cleanFrom) + if err != nil { + return "", err + } + } + remoteIP, ok := msgMeta.Conn.RemoteAddr.(*net.TCPAddr) + if !ok { + remoteIP = &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)} + } + if err := s.endp.limits.TakeMsg(context.Background(), remoteIP.IP, domain); err != nil { + return "", err + } + + s.msgCtx, s.msgTask = trace.NewTask(ctx, "Incoming Message") + + mailCtx, mailTask := trace.NewTask(s.msgCtx, "MAIL FROM") + defer mailTask.End() + + delivery, err := s.endp.pipeline.Start(mailCtx, msgMeta, cleanFrom) + if err != nil { + s.msgCtx = nil + s.msgTask.End() + s.endp.limits.ReleaseMsg(remoteIP.IP, domain) + return msgMeta.ID, err + } + + startedSMTPTransactions.WithLabelValues(s.endp.name).Inc() + + s.msgMeta = msgMeta + s.mailFrom = cleanFrom + s.delivery = delivery + + return msgMeta.ID, nil +} + +func (s *Session) Mail(from string, opts *smtp.MailOptions) error { + if s.endp.authAlwaysRequired && s.connState.AuthUser == "" { + return smtp.ErrAuthRequired + } + + s.msgLock.Lock() + defer s.msgLock.Unlock() + + if !s.endp.deferServerReject { + // Will initialize s.msgCtx. + msgID, err := s.startDelivery(s.sessionCtx, from, *opts) + if err != nil { + if !errors.Is(err, context.DeadlineExceeded) { + s.log.Error("MAIL FROM error", err, "msg_id", msgID) + } + return s.endp.wrapErr(msgID, !opts.UTF8, "MAIL", err) + } + } + + // Keep the MAIL FROM argument for deferred startDelivery. + s.mailFrom = from + s.opts = *opts + + return nil +} + +func (s *Session) fetchRDNSName(ctx context.Context) { + defer trace.StartRegion(ctx, "rDNS fetch").End() + + tcpAddr, ok := s.connState.RemoteAddr.(*net.TCPAddr) + if !ok { + s.connState.RDNSName.Set(nil, nil) + return + } + + name, err := dns.LookupAddr(ctx, s.endp.resolver, tcpAddr.IP) + if err != nil { + dnsErr, ok := err.(*net.DNSError) + if ok && dnsErr.IsNotFound { + s.connState.RDNSName.Set(nil, nil) + return + } + + if !errors.Is(err, context.Canceled) { + // Often occurs when transaction completes before rDNS lookup and + // rDNS name was not actually needed. So do not log cancelation + // error if that's the case. + + reason, misc := exterrors.UnwrapDNSErr(err) + misc["reason"] = reason + s.log.Error("rDNS error", exterrors.WithFields(err, misc), "src_ip", s.connState.RemoteAddr) + } + s.connState.RDNSName.Set(nil, err) + return + } + + s.connState.RDNSName.Set(name, nil) +} + +func (s *Session) Rcpt(to string, opts *smtp.RcptOptions) error { + s.msgLock.Lock() + defer s.msgLock.Unlock() + + // deferServerReject = true and this is the first RCPT TO command. + if s.delivery == nil { + // If we already attempted to initialize the delivery - + // fail again. + if s.deliveryErr != nil { + s.repeatedMailErrs++ + // The deliveryErr is already wrapped. + return s.deliveryErr + } + + // It will initialize s.msgCtx. + msgID, err := s.startDelivery(s.sessionCtx, s.mailFrom, s.opts) + if err != nil { + if !errors.Is(err, context.DeadlineExceeded) { + s.log.Error("MAIL FROM error (deferred)", err, "rcpt", to, "msg_id", msgID) + } + s.deliveryErr = s.endp.wrapErr(msgID, !s.opts.UTF8, "RCPT", err) + return s.deliveryErr + } + } + + rcptCtx, rcptTask := trace.NewTask(s.msgCtx, "RCPT TO") + defer rcptTask.End() + + if err := s.rcpt(rcptCtx, to, opts); err != nil { + if s.loggedRcptErrors < s.endp.maxLoggedRcptErrors { + s.log.Error("RCPT error", err, "rcpt", to, "msg_id", s.msgMeta.ID) + s.loggedRcptErrors++ + if s.loggedRcptErrors == s.endp.maxLoggedRcptErrors { + s.log.Msg("too many RCPT errors, possible dictonary attack", "src_ip", s.connState.RemoteAddr, "msg_id", s.msgMeta.ID) + } + } + return s.endp.wrapErr(s.msgMeta.ID, !s.opts.UTF8, "RCPT", err) + } + s.endp.Log.Msg("RCPT ok", "rcpt", to, "msg_id", s.msgMeta.ID) + return nil +} + +func (s *Session) rcpt(ctx context.Context, to string, opts *smtp.RcptOptions) error { + // INTERNATIONALIZATION: Do not permit non-ASCII addresses unless SMTPUTF8 is + // used. + if !address.IsASCII(to) && !s.opts.UTF8 { + return &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 6, 7}, + Message: "SMTPUTF8 is required for non-ASCII recipients", + } + } + cleanTo, err := address.CleanDomain(to) + if err != nil { + return &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 2}, + Message: "Unable to normalize the recipient address", + } + } + + return s.delivery.AddRcpt(ctx, cleanTo, *opts) +} + +func (s *Session) Logout() error { + s.msgLock.Lock() + defer s.msgLock.Unlock() + + if s.delivery != nil { + s.abort(s.msgCtx) + + if s.repeatedMailErrs > s.endp.maxLoggedRcptErrors { + s.log.Msg("MAIL FROM repeated error a lot of times, possible dictonary attack", "count", s.repeatedMailErrs, "src_ip", s.connState.RemoteAddr) + } + } + if s.cancelRDNS != nil { + s.cancelRDNS() + } + + s.endp.sessionCnt.Add(-1) + + return nil +} + +func (s *Session) prepareBody(r io.Reader) (textproto.Header, buffer.Buffer, error) { + limitr := limitReader(r, s.endp.maxHeaderBytes, &exterrors.SMTPError{ + Code: 552, + EnhancedCode: exterrors.EnhancedCode{5, 3, 4}, + Message: "Message header size exceeds limit", + }) + + bufr := bufio.NewReader(limitr) + header, err := textproto.ReadHeader(bufr) + if err != nil { + return textproto.Header{}, nil, fmt.Errorf("I/O error while parsing header: %w", err) + } + + if s.endp.submission { + // The MsgMetadata is passed by pointer all the way down. + if err := s.submissionPrepare(s.msgMeta, &header); err != nil { + return textproto.Header{}, nil, err + } + } + + // the header size check is done. The message size will be checked by go-smtp + limitr.Enabled = false + + buf, err := s.endp.buffer(bufr) + if err != nil { + return textproto.Header{}, nil, fmt.Errorf("I/O error while writing buffer: %w", err) + } + + return header, buf, nil +} + +func (s *Session) Data(r io.Reader) error { + s.msgLock.Lock() + defer s.msgLock.Unlock() + + bodyCtx, bodyTask := trace.NewTask(s.msgCtx, "DATA") + defer bodyTask.End() + + wrapErr := func(err error) error { + s.log.Error("DATA error", err, "msg_id", s.msgMeta.ID) + return s.endp.wrapErr(s.msgMeta.ID, !s.opts.UTF8, "DATA", err) + } + + header, buf, err := s.prepareBody(r) + if err != nil { + return wrapErr(err) + } + defer func() { + if err := buf.Remove(); err != nil { + s.log.Error("failed to remove buffered body", err) + } + + // go-smtp will call Reset, but it will call Abort if delivery is non-nil. + s.cleanSession() + }() + + if err := s.checkRoutingLoops(header); err != nil { + return wrapErr(err) + } + + if strings.EqualFold(header.Get("TLS-Required"), "No") { + s.msgMeta.TLSRequireOverride = true + } + + if err := s.delivery.Body(bodyCtx, header, buf); err != nil { + return wrapErr(err) + } + + if err := s.delivery.Commit(bodyCtx); err != nil { + return wrapErr(err) + } + + s.log.Msg("accepted", "msg_id", s.msgMeta.ID) + + return nil +} + +type statusWrapper struct { + sc smtp.StatusCollector + s *Session +} + +func (sw statusWrapper) SetStatus(rcpt string, err error) { + sw.sc.SetStatus(rcpt, sw.s.endp.wrapErr(sw.s.msgMeta.ID, !sw.s.opts.UTF8, "DATA", err)) +} + +func (s *Session) LMTPData(r io.Reader, sc smtp.StatusCollector) error { + s.msgLock.Lock() + defer s.msgLock.Unlock() + + bodyCtx, bodyTask := trace.NewTask(s.msgCtx, "DATA") + defer bodyTask.End() + + wrapErr := func(err error) error { + s.log.Error("DATA error", err, "msg_id", s.msgMeta.ID) + return s.endp.wrapErr(s.msgMeta.ID, !s.opts.UTF8, "DATA", err) + } + + header, buf, err := s.prepareBody(r) + if err != nil { + return wrapErr(err) + } + defer func() { + if err := buf.Remove(); err != nil { + s.log.Error("failed to remove buffered body", err) + } + + // go-smtp will call Reset, but it will call Abort if delivery is non-nil. + s.cleanSession() + }() + + if strings.EqualFold(header.Get("TLS-Required"), "No") { + s.msgMeta.TLSRequireOverride = true + } + + if err := s.checkRoutingLoops(header); err != nil { + return wrapErr(err) + } + + s.delivery.(module.PartialDelivery).BodyNonAtomic(bodyCtx, statusWrapper{sc, s}, header, buf) + + // We can't really tell whether it is failed completely or succeeded + // so always commit. Should be harmless, anyway. + if err := s.delivery.Commit(bodyCtx); err != nil { + return wrapErr(err) + } + + s.log.Msg("accepted", "msg_id", s.msgMeta.ID) + + return nil +} + +func (s *Session) checkRoutingLoops(header textproto.Header) error { + // RFC 5321 Section 6.3: + // >Simple counting of the number of "Received:" header fields in a + // >message has proven to be an effective, although rarely optimal, + // >method of detecting loops in mail systems. + receivedCount := 0 + for f := header.FieldsByKey("Received"); f.Next(); { + receivedCount++ + } + if receivedCount > s.endp.maxReceived { + return &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 4, 6}, + Message: fmt.Sprintf("Too many Received header fields (%d), possible forwarding loop", receivedCount), + } + } + + return nil +} + +func (endp *Endpoint) wrapErr(msgId string, mangleUTF8 bool, command string, err error) error { + if err == nil { + return nil + } + + if errors.Is(err, context.DeadlineExceeded) { + return &smtp.SMTPError{ + Code: 451, + EnhancedCode: smtp.EnhancedCode{4, 4, 5}, + Message: "High load, try again later", + } + } + + res := &smtp.SMTPError{ + Code: 554, + EnhancedCode: smtp.EnhancedCodeNotSet, + // Err on the side of caution if the error lacks SMTP annotations. If + // we just pass the error text through, we might accidenetally disclose + // details of server configuration. + Message: "Internal server error", + } + + if exterrors.IsTemporary(err) { + res.Code = 451 + } + + ctxInfo := exterrors.Fields(err) + ctxCode, ok := ctxInfo["smtp_code"].(int) + if ok { + res.Code = ctxCode + } + ctxEnchCode, ok := ctxInfo["smtp_enchcode"].(exterrors.EnhancedCode) + if ok { + res.EnhancedCode = smtp.EnhancedCode(ctxEnchCode) + } + ctxMsg, ok := ctxInfo["smtp_msg"].(string) + if ok { + res.Message = ctxMsg + } + + if smtpErr, ok := err.(*smtp.SMTPError); ok { + endp.Log.Printf("plain SMTP error returned, this is deprecated") + res.Code = smtpErr.Code + res.EnhancedCode = smtpErr.EnhancedCode + res.Message = smtpErr.Message + } + + if msgId != "" { + res.Message += " (msg ID = " + msgId + ")" + } + + failedCmds.WithLabelValues(endp.name, command, strconv.Itoa(res.Code), + fmt.Sprintf("%d.%d.%d", + res.EnhancedCode[0], + res.EnhancedCode[1], + res.EnhancedCode[2])).Inc() + + // INTERNATIONALIZATION: See RFC 6531 Section 3.7.4.1. + if mangleUTF8 { + b := strings.Builder{} + b.Grow(len(res.Message)) + for _, ch := range res.Message { + if ch > 128 { + b.WriteRune('?') + } else { + b.WriteRune(ch) + } + } + res.Message = b.String() + } + + return res +} diff --git a/internal/endpoint/smtp/smtp.go b/internal/endpoint/smtp/smtp.go new file mode 100644 index 0000000..8f800e5 --- /dev/null +++ b/internal/endpoint/smtp/smtp.go @@ -0,0 +1,429 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import ( + "bytes" + "context" + "crypto/tls" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + tls2 "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/future" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth" + "github.com/foxcpp/maddy/internal/authz" + "github.com/foxcpp/maddy/internal/limits" + "github.com/foxcpp/maddy/internal/msgpipeline" + "github.com/foxcpp/maddy/internal/proxy_protocol" + "golang.org/x/net/idna" +) + +type Endpoint struct { + saslAuth auth.SASLAuth + serv *smtp.Server + name string + addrs []string + listeners []net.Listener + proxyProtocol *proxy_protocol.ProxyProtocol + pipeline *msgpipeline.MsgPipeline + resolver dns.Resolver + limits *limits.Group + + buffer func(r io.Reader) (buffer.Buffer, error) + + authAlwaysRequired bool + submission bool + lmtp bool + deferServerReject bool + maxLoggedRcptErrors int + maxReceived int + maxHeaderBytes int64 + + sessionCnt atomic.Int32 + + listenersWg sync.WaitGroup + + Log log.Logger +} + +func (endp *Endpoint) Name() string { + return endp.name +} + +func (endp *Endpoint) InstanceName() string { + return endp.name +} + +func New(modName string, addrs []string) (module.Module, error) { + endp := &Endpoint{ + name: modName, + addrs: addrs, + submission: modName == "submission", + lmtp: modName == "lmtp", + resolver: dns.DefaultResolver(), + buffer: buffer.BufferInMemory, + Log: log.Logger{Name: modName}, + saslAuth: auth.SASLAuth{ + Log: log.Logger{Name: modName + "/sasl"}, + }, + } + return endp, nil +} + +func (endp *Endpoint) Init(cfg *config.Map) error { + endp.serv = smtp.NewServer(endp) + endp.serv.ErrorLog = endp.Log + endp.serv.LMTP = endp.lmtp + endp.serv.EnableSMTPUTF8 = true + endp.serv.EnableREQUIRETLS = true + if err := endp.setConfig(cfg); err != nil { + return err + } + + addresses := make([]config.Endpoint, 0, len(endp.addrs)) + for _, addr := range endp.addrs { + saddr, err := config.ParseEndpoint(addr) + if err != nil { + return fmt.Errorf("%s: invalid address: %s", addr, endp.name) + } + + addresses = append(addresses, saddr) + } + + if err := endp.setupListeners(addresses); err != nil { + for _, l := range endp.listeners { + l.Close() + } + return err + } + + allLocal := true + for _, addr := range addresses { + if addr.Scheme != "unix" && !strings.HasPrefix(addr.Host, "127.0.0.") { + allLocal = false + } + } + + if endp.serv.AllowInsecureAuth && !allLocal { + endp.Log.Println("authentication over unencrypted connections is allowed, this is insecure configuration and should be used only for testing!") + } + if endp.serv.TLSConfig == nil { + if !allLocal { + endp.Log.Println("TLS is disabled, this is insecure configuration and should be used only for testing!") + } + + endp.serv.AllowInsecureAuth = true + } + + return nil +} + +func autoBufferMode(maxSize int, dir string) func(io.Reader) (buffer.Buffer, error) { + return func(r io.Reader) (buffer.Buffer, error) { + // First try to read up to N bytes. + initial := make([]byte, maxSize) + actualSize, err := io.ReadFull(r, initial) + if err != nil { + if err == io.ErrUnexpectedEOF { + log.Debugln("autobuffer: keeping the message in RAM (read", actualSize, "bytes, got EOF)") + return buffer.MemoryBuffer{Slice: initial[:actualSize]}, nil + } + if err == io.EOF { + // Special case: message with empty body. + return buffer.MemoryBuffer{}, nil + } + // Some I/O error happened, bail out. + return nil, err + } + if actualSize < maxSize { + // Ok, the message is smaller than N. Make a MemoryBuffer and + // handle it in RAM. + log.Debugln("autobuffer: keeping the message in RAM (read", actualSize, "bytes, got short read)") + return buffer.MemoryBuffer{Slice: initial[:actualSize]}, nil + } + + log.Debugln("autobuffer: spilling the message to the FS") + // The message is big. Dump what we got to the disk and continue writing it there. + return buffer.BufferInFile( + io.MultiReader(bytes.NewReader(initial[:actualSize]), r), + dir) + } +} + +func bufferModeDirective(_ *config.Map, node config.Node) (interface{}, error) { + if len(node.Args) < 1 { + return nil, config.NodeErr(node, "at least one argument required") + } + switch node.Args[0] { + case "ram": + if len(node.Args) > 1 { + return nil, config.NodeErr(node, "no additional arguments for 'ram' mode") + } + return buffer.BufferInMemory, nil + case "fs": + path := filepath.Join(config.StateDirectory, "buffer") + if err := os.MkdirAll(path, 0o700); err != nil { + return nil, err + } + switch len(node.Args) { + case 2: + path = node.Args[1] + fallthrough + case 1: + return func(r io.Reader) (buffer.Buffer, error) { + return buffer.BufferInFile(r, path) + }, nil + default: + return nil, config.NodeErr(node, "too many arguments for 'fs' mode") + } + case "auto": + path := filepath.Join(config.StateDirectory, "buffer") + if err := os.MkdirAll(path, 0o700); err != nil { + return nil, err + } + + maxSize := 1 * 1024 * 1024 // 1 MiB + switch len(node.Args) { + case 3: + path = node.Args[2] + fallthrough + case 2: + var err error + maxSize, err = config.ParseDataSize(node.Args[1]) + if err != nil { + return nil, config.NodeErr(node, "%v", err) + } + fallthrough + case 1: + return autoBufferMode(maxSize, path), nil + default: + return nil, config.NodeErr(node, "too many arguments for 'auto' mode") + } + default: + return nil, config.NodeErr(node, "unknown buffer mode: %v", node.Args[0]) + } +} + +func (endp *Endpoint) setConfig(cfg *config.Map) error { + var ( + hostname string + err error + ioDebug bool + ) + + cfg.Callback("auth", func(m *config.Map, node config.Node) error { + return endp.saslAuth.AddProvider(m, node) + }) + cfg.Bool("sasl_login", false, false, &endp.saslAuth.EnableLogin) + cfg.String("hostname", true, true, "", &hostname) + config.EnumMapped(cfg, "auth_map_normalize", true, false, authz.NormalizeFuncs, authz.NormalizeAuto, + &endp.saslAuth.AuthNormalize) + modconfig.Table(cfg, "auth_map", true, false, nil, &endp.saslAuth.AuthMap) + cfg.Duration("write_timeout", false, false, 1*time.Minute, &endp.serv.WriteTimeout) + cfg.Duration("read_timeout", false, false, 10*time.Minute, &endp.serv.ReadTimeout) + cfg.DataSize("max_message_size", false, false, 32*1024*1024, &endp.serv.MaxMessageBytes) + cfg.DataSize("max_header_size", false, false, 1*1024*1024, &endp.maxHeaderBytes) + cfg.Int("max_recipients", false, false, 20000, &endp.serv.MaxRecipients) + cfg.Int("max_received", false, false, 50, &endp.maxReceived) + cfg.Custom("buffer", false, false, func() (interface{}, error) { + path := filepath.Join(config.StateDirectory, "buffer") + if err := os.MkdirAll(path, 0o700); err != nil { + return nil, err + } + return autoBufferMode(1*1024*1024 /* 1 MiB */, path), nil + }, bufferModeDirective, &endp.buffer) + cfg.Custom("tls", true, endp.name != "lmtp", nil, tls2.TLSDirective, &endp.serv.TLSConfig) + cfg.Custom("proxy_protocol", false, false, nil, proxy_protocol.ProxyProtocolDirective, &endp.proxyProtocol) + cfg.Bool("insecure_auth", endp.name == "lmtp", false, &endp.serv.AllowInsecureAuth) + cfg.Int("smtp_max_line_length", false, false, 4000, &endp.serv.MaxLineLength) + cfg.Bool("io_debug", false, false, &ioDebug) + cfg.Bool("debug", true, false, &endp.Log.Debug) + cfg.Bool("defer_sender_reject", false, true, &endp.deferServerReject) + cfg.Int("max_logged_rcpt_errors", false, false, 5, &endp.maxLoggedRcptErrors) + cfg.Custom("limits", false, false, func() (interface{}, error) { + return &limits.Group{}, nil + }, func(cfg *config.Map, n config.Node) (interface{}, error) { + var g *limits.Group + if err := modconfig.GroupFromNode("limits", n.Args, n, cfg.Globals, &g); err != nil { + return nil, err + } + return g, nil + }, &endp.limits) + cfg.AllowUnknown() + unknown, err := cfg.Process() + if err != nil { + return err + } + + endp.saslAuth.Log.Debug = endp.Log.Debug + + // INTERNATIONALIZATION: See RFC 6531 Section 3.3. + endp.serv.Domain, err = idna.ToASCII(hostname) + if err != nil { + return fmt.Errorf("%s: cannot represent the hostname as an A-label name: %w", endp.name, err) + } + + endp.pipeline, err = msgpipeline.New(cfg.Globals, unknown) + if err != nil { + return err + } + endp.pipeline.Hostname = endp.serv.Domain + endp.pipeline.Resolver = endp.resolver + endp.pipeline.Log = log.Logger{Name: "smtp/pipeline", Debug: endp.Log.Debug} + endp.pipeline.FirstPipeline = true + + if endp.submission { + endp.authAlwaysRequired = true + if len(endp.saslAuth.SASLMechanisms()) == 0 { + return fmt.Errorf("%s: auth. provider must be set for submission endpoint", endp.name) + } + } + + if ioDebug { + endp.serv.Debug = endp.Log.DebugWriter() + endp.Log.Println("I/O debugging is on! It may leak passwords in logs, be careful!") + } + + return nil +} + +func (endp *Endpoint) setupListeners(addresses []config.Endpoint) error { + for _, addr := range addresses { + var l net.Listener + var err error + l, err = net.Listen(addr.Network(), addr.Address()) + if err != nil { + return fmt.Errorf("%s: %w", endp.name, err) + } + endp.Log.Printf("listening on %v", addr) + + if addr.IsTLS() { + if endp.serv.TLSConfig == nil { + return fmt.Errorf("%s: can't bind on SMTPS endpoint without TLS configuration", endp.name) + } + l = tls.NewListener(l, endp.serv.TLSConfig) + } + + if endp.proxyProtocol != nil { + l = proxy_protocol.NewListener(l, endp.proxyProtocol, endp.Log) + } + + endp.listeners = append(endp.listeners, l) + + endp.listenersWg.Add(1) + go func() { + if err := endp.serv.Serve(l); err != nil { + endp.Log.Printf("failed to serve %s: %s", addr, err) + } + endp.listenersWg.Done() + }() + } + + return nil +} + +func (endp *Endpoint) NewSession(conn *smtp.Conn) (smtp.Session, error) { + sess := endp.newSession(conn) + + // Executed before authentication and session initialization. + if err := endp.pipeline.RunEarlyChecks(context.TODO(), &sess.connState); err != nil { + if err := sess.Logout(); err != nil { + endp.Log.Error("early checks logout failed", err) + } + return nil, endp.wrapErr("", true, "EHLO", err) + } + + endp.sessionCnt.Add(1) + + return sess, nil +} + +func (endp *Endpoint) newSession(conn *smtp.Conn) *Session { + s := &Session{ + endp: endp, + log: endp.Log, + sessionCtx: context.Background(), + } + + // Used in tests. + if conn == nil { + return s + } + + s.connState = module.ConnState{ + Hostname: conn.Hostname(), + LocalAddr: conn.Conn().LocalAddr(), + RemoteAddr: conn.Conn().RemoteAddr(), + } + if tlsState, ok := conn.TLSConnectionState(); ok { + s.connState.TLS = tlsState + } + + if endp.serv.LMTP { + s.connState.Proto = "LMTP" + } else { + // Check if TLS connection conn struct is poplated. + // If it is - we are ssing TLS. + if s.connState.TLS.HandshakeComplete { + s.connState.Proto = "ESMTPS" + } else { + s.connState.Proto = "ESMTP" + } + } + + if endp.resolver != nil { + rdnsCtx, cancelRDNS := context.WithCancel(s.sessionCtx) + s.connState.RDNSName = future.New() + s.cancelRDNS = cancelRDNS + go s.fetchRDNSName(rdnsCtx) + } + + return s +} + +func (endp *Endpoint) ConnectionCount() int { + return int(endp.sessionCnt.Load()) +} + +func (endp *Endpoint) Close() error { + endp.serv.Close() + endp.listenersWg.Wait() + return nil +} + +func init() { + module.RegisterEndpoint("smtp", New) + module.RegisterEndpoint("submission", New) + module.RegisterEndpoint("lmtp", New) +} diff --git a/internal/endpoint/smtp/smtp_test.go b/internal/endpoint/smtp/smtp_test.go new file mode 100644 index 0000000..fa46c27 --- /dev/null +++ b/internal/endpoint/smtp/smtp_test.go @@ -0,0 +1,591 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import ( + "flag" + "math/rand" + "net" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/emersion/go-sasl" + "github.com/emersion/go-smtp" + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth" + "github.com/foxcpp/maddy/internal/msgpipeline" + "github.com/foxcpp/maddy/internal/testutils" +) + +var testPort string + +const testMsg = "From: \r\n" + + "Subject: Hello there!\r\n" + + "\r\n" + + "foobar\r\n" + +func testEndpoint(t *testing.T, modName string, authMod module.PlainAuth, tgt module.DeliveryTarget, checks []module.Check, cfg []config.Node) *Endpoint { + t.Helper() + + mod, err := New(modName, []string{"tcp://127.0.0.1:" + testPort}) + if err != nil { + t.Fatal(err) + } + endp := mod.(*Endpoint) + + endp.resolver = &mockdns.Resolver{ + Zones: map[string]mockdns.Zone{ + "mx.example.org.": { + A: []string{"127.0.0.1"}, + }, + "1.0.0.127.in-addr.arpa.": { + PTR: []string{"mx.example.org"}, + }, + }, + } + endp.Log = testutils.Logger(t, "smtp") + + cfg = append(cfg, + config.Node{ + Name: "hostname", + Args: []string{"mx.example.com"}, + }, + config.Node{ + Name: "tls", + Args: []string{"off"}, + }, + config.Node{ // To make it succeed, pipeline is actually replaced below. + Name: "deliver_to", + Args: []string{"dummy"}, + }, + ) + + if authMod != nil { + cfg = append(cfg, config.Node{ + Name: "auth", + Args: []string{"dummy"}, + }) + } + + err = endp.Init(config.NewMap(nil, config.Node{ + Children: cfg, + })) + if err != nil { + t.Fatal(err) + } + + endp.saslAuth = auth.SASLAuth{ + Log: testutils.Logger(t, "smtp/saslauth"), + Plain: []module.PlainAuth{authMod}, + } + + endp.pipeline = msgpipeline.Mock(tgt, checks) + endp.pipeline.Hostname = "mx.example.com" + endp.pipeline.Resolver = endp.resolver + endp.pipeline.FirstPipeline = true + endp.pipeline.Log = testutils.Logger(t, "smtp/pipeline") + + return endp +} + +func submitMsg(t *testing.T, cl *smtp.Client, from string, rcpts []string, msg string) error { + return submitMsgOpts(t, cl, from, rcpts, nil, msg) +} + +func submitMsgOpts(t *testing.T, cl *smtp.Client, from string, rcpts []string, opts *smtp.MailOptions, msg string) error { + t.Helper() + + // Error for this one is ignored because it fails if EHLO was already sent + // and submitMsg can happen multiple times. + _ = cl.Hello("mx.example.org") + if err := cl.Mail(from, opts); err != nil { + return err + } + for _, rcpt := range rcpts { + if err := cl.Rcpt(rcpt, &smtp.RcptOptions{}); err != nil { + return err + } + } + data, err := cl.Data() + if err != nil { + return err + } + if _, err := data.Write([]byte(msg)); err != nil { + return err + } + + return data.Close() +} + +func TestSMTPDelivery(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsg(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + msgID := testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, "") + + receivedPrefix := `from mx.example.org (mx.example.org [127.0.0.1]) by mx.example.com (envelope-sender ) with ESMTP id ` + msgID + + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } + + if msg.MsgMeta.Conn.Proto != "ESMTP" { + t.Error("Wrong SrcProto:", msg.MsgMeta.Conn.Proto) + } + + rdnsName, _ := msg.MsgMeta.Conn.RDNSName.Get() + if rdnsName, _ := rdnsName.(string); rdnsName != "mx.example.org" { + t.Error("Wrong rDNS name:", rdnsName) + } +} + +func TestSMTPDelivery_rDNSError(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + + endp.resolver.(*mockdns.Resolver).Zones["1.0.0.127.in-addr.arpa."] = mockdns.Zone{ + Err: &net.DNSError{ + Name: "1.0.0.127.in-addr.arpa.", + Server: "127.0.0.1:53", + Err: "bad", + IsNotFound: false, + }, + } + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsg(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, "") + + rdnsName, err := msg.MsgMeta.Conn.RDNSName.Get() + if rdnsName != nil || err == nil { + t.Errorf("Wrong rDNS result: %#+v (%v)", rdnsName, err) + } +} + +func TestSMTPDelivery_EarlyCheck_Fail(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, []module.Check{ + &testutils.Check{ + EarlyErr: &exterrors.SMTPError{ + Code: 523, + Message: "Hey", + }, + }, + }, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = cl.Mail("sender@example.org", nil) + if err == nil { + t.Fatal("Expected an error, got none") + } + + smtpErr, ok := err.(*smtp.SMTPError) + if !ok { + t.Fatal("Non-SMTPError returned") + } + + if smtpErr.Code != 523 { + t.Fatal("Wrong SMTP code:", smtpErr.Code) + } + if smtpErr.Message != "Hey" { + t.Fatal("Wrong SMTP message:", smtpErr.Message) + } +} + +func TestSMTPDeliver_CheckError(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, []module.Check{ + &testutils.Check{ + ConnRes: module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 523, + Message: "Hey", + }, + Reject: true, + }, + }, + }, nil) + endp.deferServerReject = false + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = cl.Mail("sender@example.org", nil) + if err == nil { + t.Fatal("Expected an error, got none") + } + smtpErr, ok := err.(*smtp.SMTPError) + if !ok { + t.Fatal("Non-SMTPError returned") + } + + if smtpErr.Code != 523 { + t.Fatal("Wrong SMTP code:", smtpErr.Code) + } + if !strings.HasPrefix(smtpErr.Message, "Hey") { + t.Fatal("Wrong SMTP message:", smtpErr.Message) + } +} + +func TestSMTPDeliver_CheckError_Deferred(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, []module.Check{ + &testutils.Check{ + ConnRes: module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 523, + Message: "Hey", + }, + Reject: true, + }, + }, + }, nil) + endp.deferServerReject = true + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = cl.Mail("sender@example.org", nil) + if err != nil { + t.Fatal(err) + } + + checkErr := func(err error) { + if err == nil { + t.Fatal("Expected an error, got none") + } + smtpErr, ok := err.(*smtp.SMTPError) + if !ok { + t.Error("Non-SMTPError returned") + return + } + + if smtpErr.Code != 523 { + t.Error("Wrong SMTP code:", smtpErr.Code) + } + if !strings.HasPrefix(smtpErr.Message, "Hey") { + t.Error("Wrong SMTP message:", smtpErr.Message) + } + } + + checkErr(cl.Rcpt("test1@example.org", &smtp.RcptOptions{})) + checkErr(cl.Rcpt("test1@example.org", &smtp.RcptOptions{})) + checkErr(cl.Rcpt("test2@example.org", &smtp.RcptOptions{})) +} + +func TestSMTPDelivery_Multi(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsg(t, cl, "sender1@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) + if err != nil { + t.Fatal(err) + } + err = submitMsg(t, cl, "sender2@example.org", []string{"rcpt3@example.com", "rcpt4@example.com"}, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 2 { + t.Fatal("Expected two messages, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + msgID := testutils.CheckMsgID(t, &msg, "sender1@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, "") + receivedPrefix := `from mx.example.org (mx.example.org [127.0.0.1]) by mx.example.com (envelope-sender ) with ESMTP id ` + msgID + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } + + msg = tgt.Messages[1] + msgID = testutils.CheckMsgID(t, &msg, "sender2@example.org", []string{"rcpt3@example.com", "rcpt4@example.com"}, "") + receivedPrefix = `from mx.example.org (mx.example.org [127.0.0.1]) by mx.example.com (envelope-sender ) with ESMTP id ` + msgID + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } +} + +func TestSMTPDelivery_AbortData(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Hello("mx.example.org"); err != nil { + t.Fatal(err) + } + if err := cl.Mail("sender@example.org", nil); err != nil { + t.Fatal(err) + } + if err := cl.Rcpt("test@example.com", &smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + data, err := cl.Data() + if err != nil { + t.Fatal(err) + } + if _, err := data.Write([]byte(testMsg)); err != nil { + t.Fatal(err) + } + + // Then.. Suddenly, close the connection without sending the final dot. + cl.Close() + + time.Sleep(250 * time.Millisecond) + + if len(tgt.Messages) != 0 { + t.Fatal("Expected no messages, got", len(tgt.Messages)) + } +} + +func TestSMTPDelivery_EmptyMessage(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Hello("mx.example.org"); err != nil { + t.Fatal(err) + } + if err := cl.Mail("sender@example.org", nil); err != nil { + t.Fatal(err) + } + if err := cl.Rcpt("test@example.com", &smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + data, err := cl.Data() + if err != nil { + t.Fatal(err) + } + if err := data.Close(); err != nil { + t.Fatal(err) + } + + time.Sleep(250 * time.Millisecond) + + if len(tgt.Messages) != 1 { + t.Fatal("Expected 1 message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + if len(msg.Body) != 0 { + t.Fatal("Expected an empty body, got", len(msg.Body)) + } +} + +func TestSMTPDelivery_AbortLogout(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Hello("mx.example.org"); err != nil { + t.Fatal(err) + } + if err := cl.Mail("sender@example.org", nil); err != nil { + t.Fatal(err) + } + if err := cl.Rcpt("test@example.com", &smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + // Then.. Suddenly, close the connection. + cl.Close() + + time.Sleep(250 * time.Millisecond) + + if len(tgt.Messages) != 0 { + t.Fatal("Expected no messages, got", len(tgt.Messages)) + } +} + +func TestSMTPDelivery_Reset(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Mail("from-garbage@example.org", nil); err != nil { + t.Fatal(err) + } + if err := cl.Rcpt("to-garbage@example.org", &smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + if err := cl.Reset(); err != nil { + t.Fatal(err) + } + + // then submit the message as if nothing happened. + + err = submitMsg(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, "") +} + +func TestSMTPDelivery_SubmissionAuthRequire(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "submission", &module.Dummy{}, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Mail("from-garbage@example.org", nil); err == nil { + t.Fatal("Expected an error, got none") + } +} + +func TestSMTPDelivery_SubmissionAuthOK(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "submission", &module.Dummy{}, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Auth(sasl.NewPlainClient("", "user", "password")); err != nil { + t.Fatal(err) + } + + if err := submitMsg(t, cl, "sender@example.org", []string{"rcpt@example.org"}, testMsg); err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + msgID := testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt@example.org"}, "") + + if msg.MsgMeta.Conn.AuthUser != "user" { + t.Error("Wrong AuthUser:", msg.MsgMeta.Conn.AuthUser) + } + if msg.MsgMeta.Conn.AuthPassword != "password" { + t.Error("Wrong AuthPassword:", msg.MsgMeta.Conn.AuthPassword) + } + + receivedPrefix := `by mx.example.com (envelope-sender ) with ESMTP id ` + msgID + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } + + if msg.Header.Get("Message-ID") == "" { + t.Error("No submissionPrepare run") + } +} + +func TestMain(m *testing.M) { + remoteSmtpPort := flag.String("test.smtpport", "random", "(maddy) SMTP port to use for connections in tests") + flag.Parse() + + if *remoteSmtpPort == "random" { + *remoteSmtpPort = strconv.Itoa(rand.Intn(65536-10000) + 10000) + } + + testPort = *remoteSmtpPort + os.Exit(m.Run()) +} diff --git a/internal/endpoint/smtp/smtputf8_test.go b/internal/endpoint/smtp/smtputf8_test.go new file mode 100644 index 0000000..b3f5701 --- /dev/null +++ b/internal/endpoint/smtp/smtputf8_test.go @@ -0,0 +1,361 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import ( + "strings" + "testing" + + "github.com/emersion/go-smtp" + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestSMTPUTF8_MangleStatusMessage(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, []module.Check{ + &testutils.Check{ + ConnRes: module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 523, + Message: "Hey 凱凱", + }, + Reject: true, + }, + }, + }, nil) + endp.deferServerReject = false + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = cl.Mail("sender@example.org", nil) + if err == nil { + t.Fatal("Expected an error, got none") + } + smtpErr, ok := err.(*smtp.SMTPError) + if !ok { + t.Fatal("Non-SMTPError returned") + } + + if smtpErr.Code != 523 { + t.Fatal("Wrong SMTP code:", smtpErr.Code) + } + if !strings.HasPrefix(smtpErr.Message, "Hey ??") { + t.Fatal("Wrong SMTP message:", smtpErr.Message) + } +} + +func TestSMTP_RejectNonASCIIFrom(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + endp.deferServerReject = false + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsg(t, cl, "ѣ@example.org", []string{"rcpt@example.com"}, testMsg) + + smtpErr, ok := err.(*smtp.SMTPError) + if !ok { + t.Fatal("Non-SMTPError returned") + } + if smtpErr.Code != 550 { + t.Fatal("Wrong SMTP code:", smtpErr.Code) + } + if smtpErr.EnhancedCode != (smtp.EnhancedCode{5, 6, 7}) { + t.Fatal("Wrong SMTP ench. code:", smtpErr.EnhancedCode) + } +} + +func TestSMTPUTF8_NormalizeCaseFoldFrom(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + endp.deferServerReject = false + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsgOpts(t, cl, "foo@E\u0301.example.org", []string{"rcpt@example.com"}, &smtp.MailOptions{ + UTF8: true, + }, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + testutils.CheckMsgID(t, &msg, "foo@é.example.org", []string{"rcpt@example.com"}, "") +} + +func TestSMTP_RejectNonASCIIRcpt(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + endp.deferServerReject = false + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsg(t, cl, "x@example.org", []string{"ѣ@example.org"}, testMsg) + + smtpErr, ok := err.(*smtp.SMTPError) + if !ok { + t.Fatal("Non-SMTPError returned") + } + if smtpErr.Code != 553 { + t.Fatal("Wrong SMTP code:", smtpErr.Code) + } + if smtpErr.EnhancedCode != (smtp.EnhancedCode{5, 6, 7}) { + t.Fatal("Wrong SMTP ench. code:", smtpErr.EnhancedCode) + } +} + +func TestSMTPUTF8_NormalizeCaseFoldRcpt(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + endp.deferServerReject = false + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsgOpts(t, cl, "x@example.org", []string{"foo@E\u0301.example.org"}, &smtp.MailOptions{ + UTF8: true, + }, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + testutils.CheckMsgID(t, &msg, "x@example.org", []string{"foo@é.example.org"}, "") +} + +func TestSMTPUTF8_NoMangleStatusMessage(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, []module.Check{ + &testutils.Check{ + ConnRes: module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 523, + Message: "Hey 凱凱", + }, + Reject: true, + }, + }, + }, nil) + endp.deferServerReject = false + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = cl.Mail("sender@example.org", &smtp.MailOptions{ + UTF8: true, + }) + if err == nil { + t.Fatal("Expected an error, got none") + } + smtpErr, ok := err.(*smtp.SMTPError) + if !ok { + t.Fatal("Non-SMTPError returned") + } + + if smtpErr.Code != 523 { + t.Fatal("Wrong SMTP code:", smtpErr.Code) + } + if !strings.HasPrefix(smtpErr.Message, "Hey 凱凱") { + t.Fatal("Wrong SMTP message:", smtpErr.Message) + } +} + +func TestSMTPUTF8_Received_EHLO_ALabel(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Hello("凱凱.invalid"); err != nil { + t.Fatal(err) + } + + err = submitMsg(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + msgID := testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, "") + + receivedPrefix := `from xn--y9qa.invalid (mx.example.org [127.0.0.1]) by mx.example.com (envelope-sender ) with ESMTP id ` + msgID + + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } +} + +func TestSMTPUTF8_Received_rDNS_ALabel(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + endp.resolver.(*mockdns.Resolver).Zones["1.0.0.127.in-addr.arpa."] = mockdns.Zone{ + PTR: []string{"凱凱.invalid."}, + } + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsg(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + msgID := testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, "") + + receivedPrefix := `from mx.example.org (xn--y9qa.invalid [127.0.0.1]) by mx.example.com (envelope-sender ) with ESMTP id ` + msgID + + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } +} + +func TestSMTPUTF8_Received_rDNS_ULabel(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + endp.resolver.(*mockdns.Resolver).Zones["1.0.0.127.in-addr.arpa."] = mockdns.Zone{ + PTR: []string{"凱凱.invalid."}, + } + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsgOpts(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, &smtp.MailOptions{ + UTF8: true, + }, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + msgID := testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, "") + + receivedPrefix := `from mx.example.org (凱凱.invalid [127.0.0.1]) by mx.example.com (envelope-sender ) with UTF8ESMTP id ` + msgID + + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } +} + +func TestSMTPUTF8_Received_EHLO_ULabel(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Hello("凱凱.invalid"); err != nil { + t.Fatal(err) + } + + err = submitMsgOpts(t, cl, "sender@example.org", []string{"rcpt@example.com"}, &smtp.MailOptions{ + UTF8: true, + }, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + msgID := testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt@example.com"}, "") + + // Also, 'with UTF8ESMTP'. + receivedPrefix := `from 凱凱.invalid (mx.example.org [127.0.0.1]) by mx.example.com (envelope-sender ) with UTF8ESMTP id ` + msgID + + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } +} diff --git a/internal/endpoint/smtp/submission.go b/internal/endpoint/smtp/submission.go new file mode 100644 index 0000000..316c8b6 --- /dev/null +++ b/internal/endpoint/smtp/submission.go @@ -0,0 +1,148 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import ( + "errors" + "fmt" + "net/mail" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/google/uuid" +) + +var ( + msgIDField = func() (string, error) { + id, err := uuid.NewRandom() + if err != nil { + return "", err + } + return id.String(), nil + } + + now = time.Now +) + +func (s *Session) submissionPrepare(msgMeta *module.MsgMetadata, header *textproto.Header) error { + msgMeta.DontTraceSender = true + + if header.Get("Message-ID") == "" { + msgId, err := msgIDField() + if err != nil { + return errors.New("Message-ID generation failed") + } + s.log.Msg("adding missing Message-ID") + header.Set("Message-ID", "<"+msgId+"@"+s.endp.serv.Domain+">") + } + + if header.Get("From") == "" { + return &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 6, 0}, + Message: "Message does not contains a From header field", + Misc: map[string]interface{}{ + "modifier": "submission_prepare", + }, + } + } + + for _, hdr := range [...]string{"Sender"} { + if value := header.Get(hdr); value != "" { + if _, err := mail.ParseAddress(value); err != nil { + return &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 6, 0}, + Message: fmt.Sprintf("Invalid address in %s", hdr), + Misc: map[string]interface{}{ + "modifier": "submission_prepare", + "addr": value, + }, + Err: err, + } + } + } + } + for _, hdr := range [...]string{"To", "Cc", "Bcc", "Reply-To"} { + if value := header.Get(hdr); value != "" { + if _, err := mail.ParseAddressList(value); err != nil { + return &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 6, 0}, + Message: fmt.Sprintf("Invalid address in %s", hdr), + Misc: map[string]interface{}{ + "modifier": "submission_prepare", + "addr": value, + }, + Err: err, + } + } + } + } + + addrs, err := mail.ParseAddressList(header.Get("From")) + if err != nil { + return &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 6, 0}, + Message: "Invalid address in From", + Misc: map[string]interface{}{ + "modifier": "submission_prepare", + "addr": header.Get("From"), + }, + Err: err, + } + } + + // https://tools.ietf.org/html/rfc5322#section-3.6.2 + // If From contains multiple addresses, Sender field must be present. + if len(addrs) > 1 && header.Get("Sender") == "" { + return &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 6, 0}, + Message: "Missing Sender header field", + Misc: map[string]interface{}{ + "modifier": "submission_prepare", + "from": header.Get("From"), + }, + } + } + + if dateHdr := header.Get("Date"); dateHdr != "" { + _, err := parseMessageDateTime(dateHdr) + if err != nil { + return &exterrors.SMTPError{ + Code: 554, + Message: "Malformed Date header", + Misc: map[string]interface{}{ + "modifier": "submission_prepare", + "date": dateHdr, + }, + Err: err, + } + } + } else { + s.log.Msg("adding missing Date header") + header.Set("Date", now().UTC().Format("Mon, 2 Jan 2006 15:04:05 -0700")) + } + + return nil +} diff --git a/internal/endpoint/smtp/submission_test.go b/internal/endpoint/smtp/submission_test.go new file mode 100644 index 0000000..f257364 --- /dev/null +++ b/internal/endpoint/smtp/submission_test.go @@ -0,0 +1,165 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import ( + "reflect" + "testing" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/module" +) + +func init() { + msgIDField = func() (string, error) { + return "A", nil + } + + now = func() time.Time { + return time.Unix(0, 0) + } +} + +func TestSubmissionPrepare(t *testing.T) { + test := func(hdrMap, expectedMap map[string][]string) { + t.Helper() + + hdr := textproto.Header{} + for k, v := range hdrMap { + for _, field := range v { + hdr.Add(k, field) + } + } + + endp := testEndpoint(t, "submission", &module.Dummy{}, &module.Dummy{}, nil, nil) + defer func() { + // Synchronize the endpoint initialization. + // Otherwise Close will race with Serve called by setupListeners. + cl, _ := smtp.Dial("127.0.0.1:" + testPort) + cl.Close() + + endp.Close() + }() + + session, err := endp.NewSession(nil) + if err != nil { + t.Fatal(err) + } + + err = session.(*Session).submissionPrepare(&module.MsgMetadata{}, &hdr) + if expectedMap == nil { + if err == nil { + t.Error("Expected an error, got none") + } + t.Log(err) + return + } + if expectedMap != nil && err != nil { + t.Error("Unexpected error:", err) + return + } + + resMap := make(map[string][]string) + for field := hdr.Fields(); field.Next(); { + resMap[field.Key()] = append(resMap[field.Key()], field.Value()) + } + + if !reflect.DeepEqual(expectedMap, resMap) { + t.Errorf("wrong header result\nwant %#+v\ngot %#+v", expectedMap, resMap) + } + } + + // No From field. + test(map[string][]string{}, nil) + + // Malformed From field. + test(map[string][]string{ + "From": {", \"\""}, + }, nil) + test(map[string][]string{ + "From": {" adasda"}, + }, nil) + + // Malformed Reply-To. + test(map[string][]string{ + "From": {""}, + "Reply-To": {", \"\""}, + }, nil) + + // Malformed CC. + test(map[string][]string{ + "From": {""}, + "Reply-To": {""}, + "Cc": {", \"\""}, + }, nil) + + // Malformed Sender. + test(map[string][]string{ + "From": {""}, + "Reply-To": {""}, + "Cc": {""}, + "Sender": {" asd"}, + }, nil) + + // Multiple From + no Sender. + test(map[string][]string{ + "From": {", "}, + }, nil) + + // Multiple From + valid Sender. + test(map[string][]string{ + "From": {", "}, + "Sender": {""}, + "Date": {"Fri, 22 Nov 2019 20:51:31 +0800"}, + "Message-Id": {""}, + }, map[string][]string{ + "From": {", "}, + "Sender": {""}, + "Date": {"Fri, 22 Nov 2019 20:51:31 +0800"}, + "Message-Id": {""}, + }) + + // Add missing Message-Id. + test(map[string][]string{ + "From": {""}, + "Date": {"Fri, 22 Nov 2019 20:51:31 +0800"}, + }, map[string][]string{ + "From": {""}, + "Date": {"Fri, 22 Nov 2019 20:51:31 +0800"}, + "Message-Id": {""}, + }) + + // Malformed Date. + test(map[string][]string{ + "From": {""}, + "Date": {"not a date"}, + }, nil) + + // Add missing Date. + test(map[string][]string{ + "From": {""}, + "Message-Id": {""}, + }, map[string][]string{ + "From": {""}, + "Message-Id": {""}, + "Date": {"Thu, 1 Jan 1970 00:00:00 +0000"}, + }) +} diff --git a/internal/imap_filter/command/command.go b/internal/imap_filter/command/command.go new file mode 100644 index 0000000..58146dc --- /dev/null +++ b/internal/imap_filter/command/command.go @@ -0,0 +1,207 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package command + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "regexp" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +const modName = "imap.filter.command" + +var placeholderRe = regexp.MustCompile(`{[a-zA-Z0-9_]+?}`) + +type Check struct { + instName string + log log.Logger + + cmd string + cmdArgs []string +} + +func (c *Check) IMAPFilter(accountName string, rcptTo string, msgMeta *module.MsgMetadata, hdr textproto.Header, body buffer.Buffer) (folder string, flags []string, err error) { + cmd, args := c.expandCommand(msgMeta, accountName, rcptTo, hdr) + + var buf bytes.Buffer + _ = textproto.WriteHeader(&buf, hdr) + bR, err := body.Open() + if err != nil { + return "", nil, err + } + + return c.run(cmd, args, io.MultiReader(bytes.NewReader(buf.Bytes()), bR)) +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + c := &Check{ + instName: instName, + log: log.Logger{Name: modName, Debug: log.DefaultLogger.Debug}, + } + + if len(inlineArgs) == 0 { + return nil, errors.New("command: at least one argument is required (command name)") + } + + c.cmd = inlineArgs[0] + c.cmdArgs = inlineArgs[1:] + + return c, nil +} + +func (c *Check) Name() string { + return modName +} + +func (c *Check) InstanceName() string { + return c.instName +} + +func (c *Check) Init(cfg *config.Map) error { + // Check whether the inline argument command is usable. + if _, err := exec.LookPath(c.cmd); err != nil { + return fmt.Errorf("command: %w", err) + } + + _, err := cfg.Process() + return err +} + +func (c *Check) expandCommand(msgMeta *module.MsgMetadata, accountName string, rcptTo string, hdr textproto.Header) (string, []string) { + expArgs := make([]string, len(c.cmdArgs)) + + for i, arg := range c.cmdArgs { + expArgs[i] = placeholderRe.ReplaceAllStringFunc(arg, func(placeholder string) string { + switch placeholder { + case "{auth_user}": + if msgMeta.Conn == nil { + return "" + } + return msgMeta.Conn.AuthUser + case "{source_ip}": + if msgMeta.Conn == nil { + return "" + } + tcpAddr, _ := msgMeta.Conn.RemoteAddr.(*net.TCPAddr) + if tcpAddr == nil { + return "" + } + return tcpAddr.IP.String() + case "{source_host}": + if msgMeta.Conn == nil { + return "" + } + return msgMeta.Conn.Hostname + case "{source_rdns}": + if msgMeta.Conn == nil { + return "" + } + valI, err := msgMeta.Conn.RDNSName.Get() + if err != nil { + return "" + } + if valI == nil { + return "" + } + return valI.(string) + case "{msg_id}": + return msgMeta.ID + case "{sender}": + return msgMeta.OriginalFrom + case "{rcpt_to}": + return rcptTo + case "{original_rcpt_to}": + oldestOriginalRcpt := rcptTo + for originalRcpt, ok := rcptTo, true; ok; originalRcpt, ok = msgMeta.OriginalRcpts[originalRcpt] { + oldestOriginalRcpt = originalRcpt + } + return oldestOriginalRcpt + case "{subject}": + return hdr.Get("Subject") + case "{account_name}": + return accountName + } + return placeholder + }) + } + + return c.cmd, expArgs +} + +func (c *Check) run(cmdName string, args []string, stdin io.Reader) (string, []string, error) { + c.log.Debugln("running", cmdName, args) + + cmd := exec.Command(cmdName, args...) + cmd.Stdin = stdin + stdout, err := cmd.StdoutPipe() + if err != nil { + return "", nil, err + } + + if err := cmd.Start(); err != nil { + return "", nil, err + } + + scnr := bufio.NewScanner(stdout) + var ( + folder string + flags []string + ) + if scnr.Scan() { + folder = scnr.Text() + } + for scnr.Scan() { + flags = append(flags, scnr.Text()) + } + if err := scnr.Err(); err != nil { + return "", nil, err + } + + err = cmd.Wait() + if err != nil { + if _, ok := err.(*exec.ExitError); !ok { + // If that's not ExitError, the process may still be running. We do + // not want this. + if err := cmd.Process.Signal(os.Interrupt); err != nil { + c.log.Error("failed to kill process", err) + } + } + return "", nil, err + } + + c.log.Debugf("folder: %s, extra flags: %v", folder, flags) + + return folder, flags, nil +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/imap_filter/group.go b/internal/imap_filter/group.go new file mode 100644 index 0000000..c1c5ecc --- /dev/null +++ b/internal/imap_filter/group.go @@ -0,0 +1,92 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imap_filter + +import ( + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +// Group wraps multiple modifiers and runs them serially. +// +// It is also registered as a module under 'modifiers' name and acts as a +// module group. +type Group struct { + instName string + Filters []module.IMAPFilter + log log.Logger +} + +func NewGroup(_, instName string, _, _ []string) (module.Module, error) { + return &Group{ + instName: instName, + log: log.Logger{Name: "imap_filters", Debug: log.DefaultLogger.Debug}, + }, nil +} + +func (g *Group) IMAPFilter(accountName string, rcptTo string, meta *module.MsgMetadata, hdr textproto.Header, body buffer.Buffer) (folder string, flags []string, err error) { + if g == nil { + return "", nil, nil + } + var ( + finalFolder string + finalFlags = make([]string, 0, len(g.Filters)) + ) + for _, f := range g.Filters { + folder, flags, err := f.IMAPFilter(accountName, rcptTo, meta, hdr, body) + if err != nil { + g.log.Error("IMAP filter failed", err) + continue + } + if folder != "" && finalFolder == "" { + finalFolder = folder + } + finalFlags = append(finalFlags, flags...) + } + return finalFolder, finalFlags, nil +} + +func (g *Group) Init(cfg *config.Map) error { + for _, node := range cfg.Block.Children { + mod, err := modconfig.IMAPFilter(cfg.Globals, append([]string{node.Name}, node.Args...), node) + if err != nil { + return err + } + + g.Filters = append(g.Filters, mod) + } + + return nil +} + +func (g *Group) Name() string { + return "modifiers" +} + +func (g *Group) InstanceName() string { + return g.instName +} + +func init() { + module.Register("imap_filters", NewGroup) +} diff --git a/internal/libdns/acmedns.go b/internal/libdns/acmedns.go new file mode 100644 index 0000000..cb657ee --- /dev/null +++ b/internal/libdns/acmedns.go @@ -0,0 +1,28 @@ +//go:build libdns_acmedns || libdns_all +// +build libdns_acmedns libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/acmedns" +) + +func init() { + module.Register("libdns.acmedns", func(modName, instName string, _, _ []string) (module.Module, error) { + p := acmedns.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("username", false, true, "", &p.Username) + c.String("password", false, true, "", &p.Password) + c.String("subdomain", false, true, "", &p.Subdomain) + c.String("server_url", false, true, "", &p.ServerURL) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/alidns.go b/internal/libdns/alidns.go new file mode 100644 index 0000000..cb980be --- /dev/null +++ b/internal/libdns/alidns.go @@ -0,0 +1,26 @@ +//go:build libdns_alidns || libdns_all +// +build libdns_alidns libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/alidns" +) + +func init() { + module.Register("libdns.alidns", func(modName, instName string, _, _ []string) (module.Module, error) { + p := alidns.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("key_id", false, false, "", &p.AccKeyID) + c.String("key_secret", false, false, "", &p.AccKeySecret) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/cloudflare.go b/internal/libdns/cloudflare.go new file mode 100644 index 0000000..3031953 --- /dev/null +++ b/internal/libdns/cloudflare.go @@ -0,0 +1,25 @@ +//go:build libdns_cloudflare || !libdns_separate +// +build libdns_cloudflare !libdns_separate + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/cloudflare" +) + +func init() { + module.Register("libdns.cloudflare", func(modName, instName string, _, _ []string) (module.Module, error) { + p := cloudflare.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_token", false, false, "", &p.APIToken) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/digitalocean.go b/internal/libdns/digitalocean.go new file mode 100644 index 0000000..98b77b0 --- /dev/null +++ b/internal/libdns/digitalocean.go @@ -0,0 +1,25 @@ +//go:build libdns_digitalocean || !libdns_separate +// +build libdns_digitalocean !libdns_separate + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/digitalocean" +) + +func init() { + module.Register("libdns.digitalocean", func(modName, instName string, _, _ []string) (module.Module, error) { + p := digitalocean.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_token", false, false, "", &p.APIToken) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/gandi.go b/internal/libdns/gandi.go new file mode 100644 index 0000000..62c7c2d --- /dev/null +++ b/internal/libdns/gandi.go @@ -0,0 +1,38 @@ +//go:build libdns_gandi || !libdns_separate +// +build libdns_gandi !libdns_separate + +package libdns + +import ( + "fmt" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/gandi" +) + +func init() { + module.Register("libdns.gandi", func(modName, instName string, _, _ []string) (module.Module, error) { + p := gandi.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_token", false, false, "", &p.APIToken) + c.String("personal_token", false, false, "", &p.BearerToken) + }, + afterConfig: func() error { + if p.APIToken != "" { + log.Println("libdns.gandi: api_token is deprecated, use personal_token instead (https://api.gandi.net/docs/authentication/)") + } + if p.APIToken == "" && p.BearerToken == "" { + return fmt.Errorf("libdns.gandi: either api_token or personal_token should be specified") + } + return nil + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/gcore.go b/internal/libdns/gcore.go new file mode 100644 index 0000000..01d7afb --- /dev/null +++ b/internal/libdns/gcore.go @@ -0,0 +1,34 @@ +//go:build libdns_gcore || !libdns_separate +// +build libdns_gcore !libdns_separate + +package libdns + +import ( + "fmt" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/gcore" +) + +func init() { + module.Register("libdns.gcore", func(modName, instName string, _, _ []string) (module.Module, error) { + p := gcore.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_key", false, false, "", &p.APIKey) + }, + afterConfig: func() error { + if p.APIKey == "" { + return fmt.Errorf("libdns.gcore: api_key should be specified") + } + return nil + }, + + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/googleclouddns.go b/internal/libdns/googleclouddns.go new file mode 100644 index 0000000..b59c056 --- /dev/null +++ b/internal/libdns/googleclouddns.go @@ -0,0 +1,26 @@ +//go:build libdns_googleclouddns || libdns_all +// +build libdns_googleclouddns libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/googleclouddns" +) + +func init() { + module.Register("libdns.googleclouddns", func(modName, instName string, _, _ []string) (module.Module, error) { + p := googleclouddns.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("project", false, true, "", &p.Project) + c.String("service_account_json", false, false, "", &p.ServiceAccountJSON) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/hetzner.go b/internal/libdns/hetzner.go new file mode 100644 index 0000000..b360641 --- /dev/null +++ b/internal/libdns/hetzner.go @@ -0,0 +1,25 @@ +//go:build libdns_hetzner || !libdns_separate +// +build libdns_hetzner !libdns_separate + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/hetzner" +) + +func init() { + module.Register("libdns.hetzner", func(modName, instName string, _, _ []string) (module.Module, error) { + p := hetzner.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_token", false, false, "", &p.AuthAPIToken) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/leaseweb.go b/internal/libdns/leaseweb.go new file mode 100644 index 0000000..23af12d --- /dev/null +++ b/internal/libdns/leaseweb.go @@ -0,0 +1,25 @@ +//go:build libdns_leaseweb || libdns_all +// +build libdns_leaseweb libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/leaseweb" +) + +func init() { + module.Register("libdns.leaseweb", func(modName, instName string, _, _ []string) (module.Module, error) { + p := leaseweb.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_key", false, false, "", &p.APIKey) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/metaname.go b/internal/libdns/metaname.go new file mode 100644 index 0000000..2e37ddd --- /dev/null +++ b/internal/libdns/metaname.go @@ -0,0 +1,28 @@ +//go:build libdns_metaname || libdns_all +// +build libdns_metaname libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/metaname" +) + +func init() { + module.Register("libdns.metaname", func(modName, instName string, _, _ []string) (module.Module, error) { + p := metaname.Provider{ + Endpoint: "https://metaname.net/api/1.1", + } + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_key", false, false, "", &p.APIKey) + c.String("account_ref", false, false, "", &p.AccountReference) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/namecheap.go b/internal/libdns/namecheap.go new file mode 100644 index 0000000..656ebe5 --- /dev/null +++ b/internal/libdns/namecheap.go @@ -0,0 +1,28 @@ +//go:build go1.16 +// +build go1.16 + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/namecheap" +) + +func init() { + module.Register("libdns.namecheap", func(modName, instName string, _, _ []string) (module.Module, error) { + p := namecheap.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_key", false, true, "", &p.APIKey) + c.String("api_username", false, true, "", &p.User) + c.String("endpoint", false, false, "", &p.APIEndpoint) + c.String("client_ip", false, false, "", &p.ClientIP) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/namedotcom.go b/internal/libdns/namedotcom.go new file mode 100644 index 0000000..3ce3c51 --- /dev/null +++ b/internal/libdns/namedotcom.go @@ -0,0 +1,28 @@ +//go:build libdns_namedotdom || libdns_all +// +build libdns_namedotdom libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/namedotcom" +) + +func init() { + module.Register("libdns.namedotcom", func(modName, instName string, _, _ []string) (module.Module, error) { + p := namedotcom.Provider{ + Server: "https://api.name.com", + } + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("user", false, false, "", &p.User) + c.String("token", false, false, "", &p.Token) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/provider_module.go b/internal/libdns/provider_module.go new file mode 100644 index 0000000..7556150 --- /dev/null +++ b/internal/libdns/provider_module.go @@ -0,0 +1,35 @@ +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/libdns/libdns" +) + +type ProviderModule struct { + libdns.RecordDeleter + libdns.RecordAppender + setConfig func(c *config.Map) + afterConfig func() error + + instName string + modName string +} + +func (p *ProviderModule) Init(cfg *config.Map) error { + p.setConfig(cfg) + _, err := cfg.Process() + if p.afterConfig != nil { + if err := p.afterConfig(); err != nil { + return err + } + } + return err +} + +func (p *ProviderModule) Name() string { + return p.modName +} + +func (p *ProviderModule) InstanceName() string { + return p.instName +} diff --git a/internal/libdns/rfc2136.go b/internal/libdns/rfc2136.go new file mode 100644 index 0000000..19751f6 --- /dev/null +++ b/internal/libdns/rfc2136.go @@ -0,0 +1,28 @@ +//go:build libdns_rfc2136 || libdns_all +// +build libdns_rfc2136 libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/rfc2136" +) + +func init() { + module.Register("libdns.rfc2136", func(modName, instName string, _, _ []string) (module.Module, error) { + p := rfc2136.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("key_name", false, true, "", &p.KeyName) + c.String("key", false, true, "", &p.Key) + c.String("key_alg", false, true, "", &p.KeyAlg) + c.String("server", false, true, "", &p.Server) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/route53.go b/internal/libdns/route53.go new file mode 100644 index 0000000..dd50724 --- /dev/null +++ b/internal/libdns/route53.go @@ -0,0 +1,26 @@ +//go:build libdns_route53 || libdns_all +// +build libdns_route53 libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/route53" +) + +func init() { + module.Register("libdns.route53", func(modName, instName string, _, _ []string) (module.Module, error) { + p := route53.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("secret_access_key", false, false, "", &p.SecretAccessKey) + c.String("access_key_id", false, false, "", &p.AccessKeyId) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/vultr.go b/internal/libdns/vultr.go new file mode 100644 index 0000000..9157258 --- /dev/null +++ b/internal/libdns/vultr.go @@ -0,0 +1,25 @@ +//go:build libdns_vultr || !libdns_separate +// +build libdns_vultr !libdns_separate + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/vultr" +) + +func init() { + module.Register("libdns.vultr", func(modName, instName string, _, _ []string) (module.Module, error) { + p := vultr.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_token", false, false, "", &p.APIToken) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/limits/limiters/bucket.go b/internal/limits/limiters/bucket.go new file mode 100644 index 0000000..ae653b1 --- /dev/null +++ b/internal/limits/limiters/bucket.go @@ -0,0 +1,153 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package limiters + +import ( + "context" + "sync" + "time" +) + +// BucketSet combines a group of Ls into a single key-indexed structure. +// Basically, each unique key gets its own counter. The main use case for +// BucketSet is to apply per-resource rate limiting. +// +// Amount of buckets is limited to a certain value. When the size of internal +// map is around or equal to that value, next Take call will attempt to remove +// any stale buckets from the group. If it is not possible to do so (all +// buckets are in active use), Take will return false. Alternatively, in some +// rare cases, some other (undefined) waiting Take can return false. +// +// A BucksetSet without a New function assigned is no-op: Take and TakeContext +// always succeed and Release does nothing. +type BucketSet struct { + // New function is used to construct underlying L instances. + // + // It is safe to change it only when BucketSet is not used by any + // goroutine. + New func() L + + // Time after which bucket is considered stale and can be removed from the + // set. For safe use with Rate limiter, it should be at least as twice as + // big as Rate refill interval. + ReapInterval time.Duration + + MaxBuckets int + + mLck sync.Mutex + m map[string]*struct { + r L + lastUse time.Time + } +} + +func NewBucketSet(new_ func() L, reapInterval time.Duration, maxBuckets int) *BucketSet { + return &BucketSet{ + New: new_, + ReapInterval: reapInterval, + MaxBuckets: maxBuckets, + m: map[string]*struct { + r L + lastUse time.Time + }{}, + } +} + +func (r *BucketSet) Close() { + r.mLck.Lock() + defer r.mLck.Unlock() + + for _, v := range r.m { + v.r.Close() + } +} + +func (r *BucketSet) take(key string) L { + r.mLck.Lock() + defer r.mLck.Unlock() + + if len(r.m) > r.MaxBuckets { + now := time.Now() + // Attempt to get rid of stale buckets. + for k, v := range r.m { + if v.lastUse.Sub(now) > r.ReapInterval { + // Drop the bucket, if there happen to be any waiting Take for it. + // It will return 'false', but this is fine for us since this + // whole 'reaping' process will run only when we are under a + // high load and dropping random requests in this case is a + // more or less reasonable thing to do. + v.r.Close() + delete(r.m, k) + } + } + + // Still full? E.g. all buckets are in use. + if len(r.m) > r.MaxBuckets { + return nil + } + } + + bucket, ok := r.m[key] + if !ok { + r.m[key] = &struct { + r L + lastUse time.Time + }{ + r: r.New(), + lastUse: time.Now(), + } + bucket = r.m[key] + } + r.m[key].lastUse = time.Now() + + return bucket.r +} + +func (r *BucketSet) Take(key string) bool { + if r.New == nil { + return true + } + + bucket := r.take(key) + return bucket.Take() +} + +func (r *BucketSet) Release(key string) { + if r.New == nil { + return + } + + r.mLck.Lock() + defer r.mLck.Unlock() + + bucket, ok := r.m[key] + if !ok { + return + } + bucket.r.Release() +} + +func (r *BucketSet) TakeContext(ctx context.Context, key string) error { + if r.New == nil { + return nil + } + + bucket := r.take(key) + return bucket.TakeContext(ctx) +} diff --git a/internal/limits/limiters/concurrency.go b/internal/limits/limiters/concurrency.go new file mode 100644 index 0000000..18923e8 --- /dev/null +++ b/internal/limits/limiters/concurrency.go @@ -0,0 +1,68 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package limiters + +import "context" + +// Semaphore is a convenience wrapper for a channel that implements +// semaphore-kind synchronization. +// +// If the argument given to the NewSemaphore is negative or zero, +// all methods are no-op. +type Semaphore struct { + c chan struct{} +} + +func NewSemaphore(max int) Semaphore { + return Semaphore{c: make(chan struct{}, max)} +} + +func (s Semaphore) Take() bool { + if cap(s.c) <= 0 { + return true + } + s.c <- struct{}{} + return true +} + +func (s Semaphore) TakeContext(ctx context.Context) error { + if cap(s.c) <= 0 { + return nil + } + select { + case s.c <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s Semaphore) Release() { + if cap(s.c) <= 0 { + return + } + select { + case <-s.c: + default: + panic("limiters: mismatched Release call") + } +} + +func (s Semaphore) Close() { +} diff --git a/internal/limits/limiters/limiters.go b/internal/limits/limiters/limiters.go new file mode 100644 index 0000000..9b11761 --- /dev/null +++ b/internal/limits/limiters/limiters.go @@ -0,0 +1,35 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package limiters provides a set of wrappers intended to restrict the amount +// of resources consumed by the server. +package limiters + +import "context" + +// The L interface represents a blocking limiter that has some upper bound of +// resource use and blocks when it is exceeded until enough resources are +// freed. +type L interface { + Take() bool + TakeContext(context.Context) error + Release() + + // Close frees any resources used internally by Limiter for book-keeping. + Close() +} diff --git a/internal/limits/limiters/multilimit.go b/internal/limits/limiters/multilimit.go new file mode 100644 index 0000000..d4f181c --- /dev/null +++ b/internal/limits/limiters/multilimit.go @@ -0,0 +1,69 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package limiters + +import "context" + +// MultiLimit wraps multiple L implementations into a single one, locking them +// in the specified order. +// +// It does not implement any deadlock detection or avoidance algorithms. +type MultiLimit struct { + Wrapped []L +} + +func (ml *MultiLimit) Take() bool { + for i := 0; i < len(ml.Wrapped); i++ { + if !ml.Wrapped[i].Take() { + // Acquire failed, undo acquire for all other resources we already + // got. + for _, l := range ml.Wrapped[:i] { + l.Release() + } + return false + } + } + return true +} + +func (ml *MultiLimit) TakeContext(ctx context.Context) error { + for i := 0; i < len(ml.Wrapped); i++ { + if err := ml.Wrapped[i].TakeContext(ctx); err != nil { + // Acquire failed, undo acquire for all other resources we already + // got. + for _, l := range ml.Wrapped[:i] { + l.Release() + } + return err + } + } + return nil +} + +func (ml *MultiLimit) Release() { + for _, l := range ml.Wrapped { + l.Release() + } +} + +func (ml *MultiLimit) Close() { + for _, l := range ml.Wrapped { + l.Close() + } +} diff --git a/internal/limits/limiters/rate.go b/internal/limits/limiters/rate.go new file mode 100644 index 0000000..a774187 --- /dev/null +++ b/internal/limits/limiters/rate.go @@ -0,0 +1,117 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package limiters + +import ( + "context" + "errors" + "time" +) + +var ErrClosed = errors.New("limiters: Rate bucket is closed") + +// Rate structure implements a basic rate-limiter for requests using the token +// bucket approach. +// +// Take() is expected to be called before each request. Excessive calls will +// block. Timeouts can be implemented using the TakeContext method. +// +// Rate.Close causes all waiting Take to return false. TakeContext returns +// ErrClosed in this case. +// +// If burstSize = 0, all methods are no-op and always succeed. +type Rate struct { + bucket chan struct{} + stop chan struct{} +} + +func NewRate(burstSize int, interval time.Duration) Rate { + r := Rate{ + bucket: make(chan struct{}, burstSize), + stop: make(chan struct{}), + } + + if burstSize == 0 { + return r + } + + for i := 0; i < burstSize; i++ { + r.bucket <- struct{}{} + } + + go r.fill(burstSize, interval) + return r +} + +func (r Rate) fill(burstSize int, interval time.Duration) { + t := time.NewTimer(interval) + defer t.Stop() + for { + t.Reset(interval) + select { + case <-t.C: + case <-r.stop: + close(r.bucket) + return + } + + fill: + for i := 0; i < burstSize; i++ { + select { + case r.bucket <- struct{}{}: + default: + // If there are no Take pending and the bucket is already + // full - don't block. + break fill + } + } + } +} + +func (r Rate) Take() bool { + if cap(r.bucket) == 0 { + return true + } + + _, ok := <-r.bucket + return ok +} + +func (r Rate) TakeContext(ctx context.Context) error { + if cap(r.bucket) == 0 { + return nil + } + + select { + case _, ok := <-r.bucket: + if !ok { + return ErrClosed + } + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (r Rate) Release() { +} + +func (r Rate) Close() { + close(r.stop) +} diff --git a/internal/limits/limits.go b/internal/limits/limits.go new file mode 100644 index 0000000..95d98a2 --- /dev/null +++ b/internal/limits/limits.go @@ -0,0 +1,234 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package limit provides a module object that can be used to restrict the +// concurrency and rate of the messages flow globally or on per-source, +// per-destination basis. +// +// Note, all domain inputs are interpreted with the assumption they are already +// normalized. +// +// Low-level components are available in the limiters/ subpackage. +package limits + +import ( + "context" + "net" + "strconv" + "time" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/limits/limiters" +) + +type Group struct { + instName string + + global limiters.MultiLimit + ip *limiters.BucketSet // BucketSet of MultiLimit + source *limiters.BucketSet // BucketSet of MultiLimit + dest *limiters.BucketSet // BucketSet of MultiLimit +} + +func New(_, instName string, _, _ []string) (module.Module, error) { + return &Group{ + instName: instName, + }, nil +} + +func (g *Group) Init(cfg *config.Map) error { + var ( + globalL []limiters.L + ipL []func() limiters.L + sourceL []func() limiters.L + destL []func() limiters.L + ) + + for _, child := range cfg.Block.Children { + if len(child.Args) < 1 { + return config.NodeErr(child, "at least two arguments are required") + } + + var ( + ctor func() limiters.L + err error + ) + switch kind := child.Args[0]; kind { + case "rate": + ctor, err = rateCtor(child, child.Args[1:]) + case "concurrency": + ctor, err = concurrencyCtor(child, child.Args[1:]) + default: + return config.NodeErr(child, "unknown limit kind: %v", kind) + } + if err != nil { + return err + } + + switch scope := child.Name; scope { + case "all": + globalL = append(globalL, ctor()) + case "ip": + ipL = append(ipL, ctor) + case "source": + sourceL = append(sourceL, ctor) + case "destination": + destL = append(destL, ctor) + default: + return config.NodeErr(child, "unknown limit scope: %v", scope) + } + } + + // 20010 is slightly higher than the default max. recipients count in + // endpoint/smtp. + g.global = limiters.MultiLimit{Wrapped: globalL} + if len(ipL) != 0 { + g.ip = limiters.NewBucketSet(func() limiters.L { + l := make([]limiters.L, 0, len(ipL)) + for _, ctor := range ipL { + l = append(l, ctor()) + } + return &limiters.MultiLimit{Wrapped: l} + }, 1*time.Minute, 20010) + } + if len(sourceL) != 0 { + g.source = limiters.NewBucketSet(func() limiters.L { + l := make([]limiters.L, 0, len(sourceL)) + for _, ctor := range sourceL { + l = append(l, ctor()) + } + return &limiters.MultiLimit{Wrapped: l} + }, 1*time.Minute, 20010) + } + if len(destL) != 0 { + g.dest = limiters.NewBucketSet(func() limiters.L { + l := make([]limiters.L, 0, len(sourceL)) + for _, ctor := range sourceL { + l = append(l, ctor()) + } + return &limiters.MultiLimit{Wrapped: l} + }, 1*time.Minute, 20010) + } + + return nil +} + +func rateCtor(node config.Node, args []string) (func() limiters.L, error) { + period := 1 * time.Second + burst := 0 + + switch len(args) { + case 2: + var err error + period, err = time.ParseDuration(args[1]) + if err != nil { + return nil, config.NodeErr(node, "%v", err) + } + fallthrough + case 1: + var err error + burst, err = strconv.Atoi(args[0]) + if err != nil { + return nil, config.NodeErr(node, "%v", err) + } + case 0: + return nil, config.NodeErr(node, "at least burst size is needed") + default: + return nil, config.NodeErr(node, "too many arguments") + } + + return func() limiters.L { + return limiters.NewRate(burst, period) + }, nil +} + +func concurrencyCtor(node config.Node, args []string) (func() limiters.L, error) { + if len(args) != 1 { + return nil, config.NodeErr(node, "max concurrency value is needed") + } + max, err := strconv.Atoi(args[0]) + if err != nil { + return nil, config.NodeErr(node, "%v", err) + } + return func() limiters.L { + return limiters.NewSemaphore(max) + }, nil +} + +func (g *Group) TakeMsg(ctx context.Context, addr net.IP, sourceDomain string) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + if err := g.global.TakeContext(ctx); err != nil { + return err + } + + if g.ip != nil { + if err := g.ip.TakeContext(ctx, addr.String()); err != nil { + g.global.Release() + return err + } + } + if g.source != nil { + if err := g.source.TakeContext(ctx, sourceDomain); err != nil { + g.global.Release() + g.ip.Release(addr.String()) + return err + } + } + return nil +} + +func (g *Group) TakeDest(ctx context.Context, domain string) error { + if g.dest == nil { + return nil + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + return g.dest.TakeContext(ctx, domain) +} + +func (g *Group) ReleaseMsg(addr net.IP, sourceDomain string) { + g.global.Release() + if g.ip != nil { + g.ip.Release(addr.String()) + } + if g.source != nil { + g.source.Release(sourceDomain) + } +} + +func (g *Group) ReleaseDest(domain string) { + if g.dest == nil { + return + } + g.dest.Release(domain) +} + +func (g *Group) Name() string { + return "limits" +} + +func (g *Group) InstanceName() string { + return g.instName +} + +func init() { + module.Register("limits", New) +} diff --git a/internal/modify/dkim/dkim.go b/internal/modify/dkim/dkim.go new file mode 100644 index 0000000..ffeed4a --- /dev/null +++ b/internal/modify/dkim/dkim.go @@ -0,0 +1,375 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dkim + +import ( + "context" + "crypto" + "errors" + "fmt" + "io" + "path/filepath" + "runtime/trace" + "strings" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/dkim" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" + "golang.org/x/net/idna" +) + +const Day = 86400 * time.Second + +var ( + oversignDefault = []string{ + // Directly visible to the user. + "Subject", + "Sender", + "To", + "Cc", + "From", + "Date", + + // Affects body processing. + "MIME-Version", + "Content-Type", + "Content-Transfer-Encoding", + + // Affects user interaction. + "Reply-To", + "In-Reply-To", + "Message-Id", + "References", + + // Provide additional security benefit for OpenPGP. + "Autocrypt", + "Openpgp", + } + signDefault = []string{ + // Mailing list information. Not oversigned to prevent signature + // breakage by aliasing MLMs. + "List-Id", + "List-Help", + "List-Unsubscribe", + "List-Post", + "List-Owner", + "List-Archive", + + // Not oversigned since it can be prepended by intermediate relays. + "Resent-To", + "Resent-Sender", + "Resent-Message-Id", + "Resent-Date", + "Resent-From", + "Resent-Cc", + } + + hashFuncs = map[string]crypto.Hash{ + "sha256": crypto.SHA256, + } +) + +type Modifier struct { + instName string + + domains []string + selector string + signers map[string]crypto.Signer + oversignHeader []string + signHeader []string + headerCanon dkim.Canonicalization + bodyCanon dkim.Canonicalization + sigExpiry time.Duration + hash crypto.Hash + multipleFromOk bool + signSubdomains bool + + log log.Logger +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + m := &Modifier{ + instName: instName, + signers: map[string]crypto.Signer{}, + log: log.Logger{Name: "modify.dkim"}, + } + + if len(inlineArgs) == 0 { + return m, nil + } + if len(inlineArgs) == 1 { + return nil, errors.New("modify.dkim: at least two arguments required") + } + + m.domains = inlineArgs[0 : len(inlineArgs)-1] + m.selector = inlineArgs[len(inlineArgs)-1] + + return m, nil +} + +func (m *Modifier) Name() string { + return "modify.dkim" +} + +func (m *Modifier) InstanceName() string { + return m.instName +} + +func (m *Modifier) Init(cfg *config.Map) error { + var ( + hashName string + keyPathTemplate string + newKeyAlgo string + ) + + cfg.Bool("debug", true, false, &m.log.Debug) + cfg.StringList("domains", false, false, m.domains, &m.domains) + cfg.String("selector", false, false, m.selector, &m.selector) + cfg.String("key_path", false, false, "dkim_keys/{domain}_{selector}.key", &keyPathTemplate) + cfg.StringList("oversign_fields", false, false, oversignDefault, &m.oversignHeader) + cfg.StringList("sign_fields", false, false, signDefault, &m.signHeader) + cfg.Enum("header_canon", false, false, + []string{string(dkim.CanonicalizationRelaxed), string(dkim.CanonicalizationSimple)}, + dkim.CanonicalizationRelaxed, (*string)(&m.headerCanon)) + cfg.Enum("body_canon", false, false, + []string{string(dkim.CanonicalizationRelaxed), string(dkim.CanonicalizationSimple)}, + dkim.CanonicalizationRelaxed, (*string)(&m.bodyCanon)) + cfg.Duration("sig_expiry", false, false, 5*Day, &m.sigExpiry) + cfg.Enum("hash", false, false, + []string{"sha256"}, "sha256", &hashName) + cfg.Enum("newkey_algo", false, false, + []string{"rsa4096", "rsa2048", "ed25519"}, "rsa2048", &newKeyAlgo) + cfg.Bool("allow_multiple_from", false, false, &m.multipleFromOk) + cfg.Bool("sign_subdomains", false, false, &m.signSubdomains) + + if _, err := cfg.Process(); err != nil { + return err + } + + if len(m.domains) == 0 { + return errors.New("sign_domain: at least one domain is needed") + } + if m.selector == "" { + return errors.New("sign_domain: selector is not specified") + } + if m.signSubdomains && len(m.domains) > 1 { + return errors.New("sign_domain: only one domain is supported when sign_subdomains is enabled") + } + + m.hash = hashFuncs[hashName] + if m.hash == 0 { + panic("modify.dkim.Init: Hash function allowed by config matcher but not present in hashFuncs") + } + + for _, domain := range m.domains { + if _, err := idna.ToASCII(domain); err != nil { + m.log.Printf("warning: unable to convert domain %s to A-labels form, non-EAI messages will not be signed: %v", domain, err) + } + + keyValues := strings.NewReplacer("{domain}", domain, "{selector}", m.selector) + keyPath := keyValues.Replace(keyPathTemplate) + + signer, newKey, err := m.loadOrGenerateKey(keyPath, newKeyAlgo) + if err != nil { + return err + } + + if newKey { + dnsPath := keyPath + ".dns" + if filepath.Ext(keyPath) == ".key" { + dnsPath = keyPath[:len(keyPath)-4] + ".dns" + } + m.log.Printf("generated a new %s keypair, private key is in %s, TXT record with public key is in %s,\n"+ + "put its contents into TXT record for %s._domainkey.%s to make signing and verification work", + newKeyAlgo, keyPath, dnsPath, m.selector, domain) + } + + normDomain, err := dns.ForLookup(domain) + if err != nil { + return fmt.Errorf("sign_skim: unable to normalize domain %s: %w", domain, err) + } + m.signers[normDomain] = signer + } + + return nil +} + +func (m *Modifier) fieldsToSign(h *textproto.Header) []string { + // Filter out duplicated fields from configs so they + // will not cause panic() in go-msgauth internals. + seen := make(map[string]struct{}) + + res := make([]string, 0, len(m.oversignHeader)+len(m.signHeader)) + for _, key := range m.oversignHeader { + if _, ok := seen[strings.ToLower(key)]; ok { + continue + } + seen[strings.ToLower(key)] = struct{}{} + + // Add to signing list once per each key use. + for field := h.FieldsByKey(key); field.Next(); { + res = append(res, key) + } + // And once more to "oversign" it. + res = append(res, key) + } + for _, key := range m.signHeader { + if _, ok := seen[strings.ToLower(key)]; ok { + continue + } + seen[strings.ToLower(key)] = struct{}{} + + // Add to signing list once per each key use. + for field := h.FieldsByKey(key); field.Next(); { + res = append(res, key) + } + } + return res +} + +type state struct { + m *Modifier + meta *module.MsgMetadata + from string + log log.Logger +} + +func (m *Modifier) ModStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.ModifierState, error) { + return &state{ + m: m, + meta: msgMeta, + log: target.DeliveryLogger(m.log, msgMeta), + }, nil +} + +func (s *state) RewriteSender(ctx context.Context, mailFrom string) (string, error) { + s.from = mailFrom + return mailFrom, nil +} + +func (s state) RewriteRcpt(ctx context.Context, rcptTo string) ([]string, error) { + return []string{rcptTo}, nil +} + +func (s *state) RewriteBody(ctx context.Context, h *textproto.Header, body buffer.Buffer) error { + defer trace.StartRegion(ctx, "modify.dkim/RewriteBody").End() + + var domain string + if s.from != "" { + var err error + _, domain, err = address.Split(s.from) + if err != nil { + return err + } + } + // Use first key for null return path (<>) and postmaster () + if domain == "" { + domain = s.m.domains[0] + } + selector := s.m.selector + + if s.m.signSubdomains { + topDomain := s.m.domains[0] + if strings.HasSuffix(domain, "."+topDomain) { + domain = topDomain + } + } + normDomain, err := dns.ForLookup(domain) + if err != nil { + s.log.Error("unable to normalize domain from envelope sender", err, "domain", domain) + return nil + } + keySigner := s.m.signers[normDomain] + if keySigner == nil { + s.log.Msg("no key for domain", "domain", normDomain) + return nil + } + + // If the message is non-EAI, we are not allowed to use domains in U-labels, + // attempt to convert. + if !s.meta.SMTPOpts.UTF8 { + var err error + domain, err = idna.ToASCII(domain) + if err != nil { + return nil + } + + selector, err = idna.ToASCII(selector) + if err != nil { + return nil + } + } + + opts := dkim.SignOptions{ + Domain: domain, + Selector: selector, + Identifier: "@" + domain, + Signer: keySigner, + Hash: s.m.hash, + HeaderCanonicalization: s.m.headerCanon, + BodyCanonicalization: s.m.bodyCanon, + HeaderKeys: s.m.fieldsToSign(h), + } + if s.m.sigExpiry != 0 { + opts.Expiration = time.Now().Add(s.m.sigExpiry) + } + signer, err := dkim.NewSigner(&opts) + if err != nil { + return exterrors.WithFields(err, map[string]interface{}{"modifier": "modify.dkim"}) + } + if err := textproto.WriteHeader(signer, *h); err != nil { + signer.Close() + return exterrors.WithFields(err, map[string]interface{}{"modifier": "modify.dkim"}) + } + r, err := body.Open() + if err != nil { + signer.Close() + return exterrors.WithFields(err, map[string]interface{}{"modifier": "modify.dkim"}) + } + if _, err := io.Copy(signer, r); err != nil { + signer.Close() + return exterrors.WithFields(err, map[string]interface{}{"modifier": "modify.dkim"}) + } + + if err := signer.Close(); err != nil { + return exterrors.WithFields(err, map[string]interface{}{"modifier": "modify.dkim"}) + } + + h.AddRaw([]byte(signer.Signature())) + + s.m.log.DebugMsg("signed", "domain", domain) + + return nil +} + +func (s state) Close() error { + return nil +} + +func init() { + module.Register("modify.dkim", New) +} diff --git a/internal/modify/dkim/dkim_test.go b/internal/modify/dkim/dkim_test.go new file mode 100644 index 0000000..d4a9b5a --- /dev/null +++ b/internal/modify/dkim/dkim_test.go @@ -0,0 +1,245 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dkim + +import ( + "bytes" + "context" + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/dkim" + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func newTestModifier(t *testing.T, dir, keyAlgo string, domains []string) *Modifier { + mod, err := New("", "test", nil, nil) + if err != nil { + t.Fatal(err) + } + m := mod.(*Modifier) + m.log = testutils.Logger(t, m.Name()) + + err = m.Init(config.NewMap(nil, config.Node{ + Children: []config.Node{ + { + Name: "domains", + Args: domains, + }, + { + Name: "selector", + Args: []string{"default"}, + }, + { + Name: "key_path", + Args: []string{filepath.Join(dir, "{domain}.key")}, + }, + { + Name: "newkey_algo", + Args: []string{keyAlgo}, + }, + }, + })) + if err != nil { + t.Fatal(err) + } + + return m +} + +func signTestMsg(t *testing.T, m *Modifier, envelopeFrom string) (textproto.Header, []byte) { + t.Helper() + + state, err := m.ModStateForMsg(context.Background(), &module.MsgMetadata{}) + if err != nil { + t.Fatal(err) + } + + testHdr := textproto.Header{} + testHdr.Add("From", "") + testHdr.Add("Subject", "heya") + testHdr.Add("To", "") + body := []byte("hello there\r\n") + + // modify.dkim expects RewriteSender to be called to get envelope sender + // (see module.Modifier docs) + + // RewriteSender does not fail for modify.dkim. It just sets envelopeFrom. + if _, err := state.RewriteSender(context.Background(), envelopeFrom); err != nil { + panic(err) + } + err = state.RewriteBody(context.Background(), &testHdr, buffer.MemoryBuffer{Slice: body}) + if err != nil { + t.Fatal(err) + } + + return testHdr, body +} + +func verifyTestMsg(t *testing.T, keysPath string, expectedDomains []string, hdr textproto.Header, body []byte) { + t.Helper() + + domainsMap := make(map[string]bool) + zones := map[string]mockdns.Zone{} + for _, domain := range expectedDomains { + dnsRecord, err := os.ReadFile(filepath.Join(keysPath, domain+".dns")) + if err != nil { + t.Fatal(err) + } + + t.Log("DNS record:", string(dnsRecord)) + zones["default._domainkey."+domain+"."] = mockdns.Zone{TXT: []string{string(dnsRecord)}} + domainsMap[domain] = false + } + + var fullBody bytes.Buffer + if err := textproto.WriteHeader(&fullBody, hdr); err != nil { + t.Fatal(err) + } + if _, err := fullBody.Write(body); err != nil { + t.Fatal(err) + } + + resolver := &mockdns.Resolver{Zones: zones} + verifs, err := dkim.VerifyWithOptions(bytes.NewReader(fullBody.Bytes()), &dkim.VerifyOptions{ + LookupTXT: func(domain string) ([]string, error) { + return resolver.LookupTXT(context.Background(), domain) + }, + }) + if err != nil { + t.Fatal(err) + } + for _, v := range verifs { + if v.Err != nil { + t.Errorf("Verification error for %s: %v", v.Domain, v.Err) + } + if _, ok := domainsMap[v.Domain]; !ok { + t.Errorf("Unexpected verification for domain %s", v.Domain) + } + + domainsMap[v.Domain] = true + } + for domain, ok := range domainsMap { + if !ok { + t.Errorf("Missing verification for domain %s", domain) + } + } +} + +func TestGenerateSignVerify(t *testing.T) { + // This test verifies whether a freshly generated key can be used for + // signing and verification. + // + // It is a kind of "integration" test for DKIM modifier, as it tests + // whether everything works correctly together. + // + // Additionally it also tests whether key selection works correctly. + + test := func(domains []string, envelopeFrom string, expectDomain []string, keyAlgo string, headerCanon, bodyCanon dkim.Canonicalization, reload bool) { + t.Helper() + + dir := t.TempDir() + + m := newTestModifier(t, dir, keyAlgo, domains) + m.bodyCanon = bodyCanon + m.headerCanon = headerCanon + if reload { + m = newTestModifier(t, dir, keyAlgo, domains) + } + + testHdr, body := signTestMsg(t, m, envelopeFrom) + verifyTestMsg(t, dir, expectDomain, testHdr, body) + } + + for _, algo := range [2]string{"rsa2048", "ed25519"} { + for _, hdrCanon := range [2]dkim.Canonicalization{dkim.CanonicalizationSimple, dkim.CanonicalizationRelaxed} { + for _, bodyCanon := range [2]dkim.Canonicalization{dkim.CanonicalizationSimple, dkim.CanonicalizationRelaxed} { + test([]string{"maddy.test"}, "test@maddy.test", []string{"maddy.test"}, algo, hdrCanon, bodyCanon, false) + test([]string{"maddy.test"}, "test@maddy.test", []string{"maddy.test"}, algo, hdrCanon, bodyCanon, true) + } + } + } + + // Key selection tests + test( + []string{"maddy.test"}, // Generated keys. + "test@maddy.test", // Envelope sender. + []string{"maddy.test"}, // Expected signature domains. + "ed25519", dkim.CanonicalizationRelaxed, dkim.CanonicalizationRelaxed, false) + test( + []string{"maddy.test"}, + "test@unrelated.maddy.test", + []string{}, + "ed25519", dkim.CanonicalizationRelaxed, dkim.CanonicalizationRelaxed, false) + test( + []string{"maddy.test", "related.maddy.test"}, + "test@related.maddy.test", + []string{"related.maddy.test"}, + "ed25519", dkim.CanonicalizationRelaxed, dkim.CanonicalizationRelaxed, false) + test( + []string{"fallback.maddy.test", "maddy.test"}, + "postmaster", + []string{"fallback.maddy.test"}, + "ed25519", dkim.CanonicalizationRelaxed, dkim.CanonicalizationRelaxed, false) + test( + []string{"fallback.maddy.test", "maddy.test"}, + "", + []string{"fallback.maddy.test"}, + "ed25519", dkim.CanonicalizationRelaxed, dkim.CanonicalizationRelaxed, false) + test( + []string{"another.maddy.test", "another.maddy.test", "maddy.test"}, + "test@another.maddy.test", + []string{"another.maddy.test"}, + "ed25519", dkim.CanonicalizationRelaxed, dkim.CanonicalizationRelaxed, false) + test( + []string{"another.maddy.test", "another.maddy.test", "maddy.test"}, + "", + []string{"another.maddy.test"}, + "ed25519", dkim.CanonicalizationRelaxed, dkim.CanonicalizationRelaxed, false) +} + +func TestFieldsToSign(t *testing.T) { + h := textproto.Header{} + h.Add("A", "1") + h.Add("c", "2") + h.Add("C", "3") + h.Add("a", "4") + h.Add("b", "5") + h.Add("unrelated", "6") + + m := Modifier{ + oversignHeader: []string{"A", "B"}, + signHeader: []string{"C"}, + } + fields := m.fieldsToSign(&h) + sort.Strings(fields) + expected := []string{"A", "A", "A", "B", "B", "C", "C"} + + if !reflect.DeepEqual(fields, expected) { + t.Errorf("incorrect set of fields to sign\nwant: %v\ngot: %v", expected, fields) + } +} diff --git a/internal/modify/dkim/keys.go b/internal/modify/dkim/keys.go new file mode 100644 index 0000000..7c39b76 --- /dev/null +++ b/internal/modify/dkim/keys.go @@ -0,0 +1,184 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dkim + +import ( + "crypto" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + "io" + "os" + "path/filepath" +) + +func (m *Modifier) loadOrGenerateKey(keyPath, newKeyAlgo string) (pkey crypto.Signer, newKey bool, err error) { + f, err := os.Open(keyPath) + if err != nil { + if os.IsNotExist(err) { + pkey, err = m.generateAndWrite(keyPath, newKeyAlgo) + return pkey, true, err + } + return nil, false, err + } + defer f.Close() + + pemBlob, err := io.ReadAll(f) + if err != nil { + return nil, false, err + } + + block, _ := pem.Decode(pemBlob) + if block == nil { + return nil, false, fmt.Errorf("modify.dkim: %s: invalid PEM block", keyPath) + } + + var key interface{} + switch block.Type { + case "PRIVATE KEY": // RFC 5208 aka PKCS #8 + key, err = x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, false, fmt.Errorf("modify.dkim: %s: %w", keyPath, err) + } + case "RSA PRIVATE KEY": // RFC 3447 aka PKCS #1 + key, err = x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, false, fmt.Errorf("modify.dkim: %s: %w", keyPath, err) + } + case "EC PRIVATE KEY": // RFC 5915 + key, err = x509.ParseECPrivateKey(block.Bytes) + if err != nil { + return nil, false, fmt.Errorf("modify.dkim: %s: %w", keyPath, err) + } + default: + return nil, false, fmt.Errorf("modify.dkim: %s: not a private key or unsupported format", keyPath) + } + + switch key := key.(type) { + case *rsa.PrivateKey: + if err := key.Validate(); err != nil { + return nil, false, err + } + key.Precompute() + return key, false, nil + case ed25519.PrivateKey: + return key, false, nil + case *ecdsa.PublicKey: + return nil, false, fmt.Errorf("modify.dkim: %s: ECDSA keys are not supported", keyPath) + default: + return nil, false, fmt.Errorf("modify.dkim: %s: unknown key type: %T", keyPath, key) + } +} + +func (m *Modifier) generateAndWrite(keyPath, newKeyAlgo string) (crypto.Signer, error) { + wrapErr := func(err error) error { + return fmt.Errorf("modify.dkim: generate %s: %w", keyPath, err) + } + + m.log.Printf("generating a new %s keypair...", newKeyAlgo) + + var ( + pkey crypto.Signer + dkimName = newKeyAlgo + err error + ) + switch newKeyAlgo { + case "rsa4096": + dkimName = "rsa" + pkey, err = rsa.GenerateKey(rand.Reader, 4096) + case "rsa2048": + dkimName = "rsa" + pkey, err = rsa.GenerateKey(rand.Reader, 2048) + case "ed25519": + _, pkey, err = ed25519.GenerateKey(rand.Reader) + default: + err = fmt.Errorf("unknown key algorithm: %s", newKeyAlgo) + } + if err != nil { + return nil, wrapErr(err) + } + + keyBlob, err := x509.MarshalPKCS8PrivateKey(pkey) + if err != nil { + return nil, wrapErr(err) + } + + // 0777 because we have public keys in here too and they don't + // need protection. Individual private key files have 0600 perms. + if err := os.MkdirAll(filepath.Dir(keyPath), 0o777); err != nil { + return nil, wrapErr(err) + } + + _, err = writeDNSRecord(keyPath, dkimName, pkey) + if err != nil { + return nil, wrapErr(err) + } + + f, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return nil, wrapErr(err) + } + + if err := pem.Encode(f, &pem.Block{ + Type: "PRIVATE KEY", + Bytes: keyBlob, + }); err != nil { + return nil, wrapErr(err) + } + + return pkey, nil +} + +func writeDNSRecord(keyPath, dkimAlgoName string, pkey crypto.Signer) (string, error) { + var ( + keyBlob []byte + pubkey = pkey.Public() + ) + switch pubkey := pubkey.(type) { + case *rsa.PublicKey: + var err error + keyBlob, err = x509.MarshalPKIXPublicKey(pubkey) + if err != nil { + return "", err + } + case ed25519.PublicKey: + keyBlob = pubkey + default: + panic("modify.dkim.writeDNSRecord: unknown key algorithm") + } + + dnsPath := keyPath + ".dns" + if filepath.Ext(keyPath) == ".key" { + dnsPath = keyPath[:len(keyPath)-4] + ".dns" + } + dnsF, err := os.Create(dnsPath) + if err != nil { + return "", err + } + keyRecord := fmt.Sprintf("v=DKIM1; k=%s; p=%s", dkimAlgoName, base64.StdEncoding.EncodeToString(keyBlob)) + if _, err := io.WriteString(dnsF, keyRecord); err != nil { + return "", err + } + return dnsPath, nil +} diff --git a/internal/modify/dkim/keys_test.go b/internal/modify/dkim/keys_test.go new file mode 100644 index 0000000..ebe13ba --- /dev/null +++ b/internal/modify/dkim/keys_test.go @@ -0,0 +1,156 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dkim + +import ( + "crypto/ed25519" + "crypto/rsa" + "encoding/base64" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestKeyLoad_new(t *testing.T) { + m := Modifier{} + m.log = testutils.Logger(t, m.Name()) + + dir := t.TempDir() + + signer, newKey, err := m.loadOrGenerateKey(filepath.Join(dir, "testkey.key"), "ed25519") + if err != nil { + t.Fatal(err) + } + if !newKey { + t.Fatal("newKey=false") + } + + recordBlob, err := os.ReadFile(filepath.Join(dir, "testkey.dns")) + if err != nil { + t.Fatal(err) + } + var keyBlob []byte + for _, part := range strings.Split(string(recordBlob), ";") { + part = strings.TrimSpace(part) + if strings.HasPrefix(part, "k=") { + if part != "k=ed25519" { + t.Fatalf("Wrong type of generated key, want ed25519, got %s", part) + } + } + if strings.HasPrefix(part, "p=") { + keyBlob, err = base64.StdEncoding.DecodeString(part[2:]) + if err != nil { + t.Fatal(err) + } + } + } + + blob := signer.Public().(ed25519.PublicKey) + if string(blob) != string(keyBlob) { + t.Fatal("wrong public key placed into record file") + } +} + +const pkeyEd25519 = `-----BEGIN PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIJG9zs4vi2MYNkL9gUQwlmBLCzDODIJ5/1CwTAZFDm5U +-----END PRIVATE KEY-----` + +const pubkeyEd25519 = `5TPcCxzVByMyRsMFs5Dx23pnxKilI+1UrGg0t+O2oZU=` + +func TestKeyLoad_existing_pkcs8(t *testing.T) { + m := Modifier{} + m.log = testutils.Logger(t, m.Name()) + + dir := t.TempDir() + + if err := os.WriteFile(filepath.Join(dir, "testkey.key"), []byte(pkeyEd25519), 0o600); err != nil { + t.Fatal(err) + } + + signer, newKey, err := m.loadOrGenerateKey(filepath.Join(dir, "testkey.key"), "ed25519") + if err != nil { + t.Fatal(err) + } + if newKey { + t.Fatal("newKey = true") + } + + blob := signer.Public().(ed25519.PublicKey) + if signerKey := base64.StdEncoding.EncodeToString(blob); signerKey != pubkeyEd25519 { + t.Fatalf("wrong public key returned by loadOrGenerateKey, \nwant %s\ngot %s", pubkeyEd25519, signerKey) + } +} + +const pkeyRSA = `-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAuxWwDR9ADiuV2b9xF+btOIgwS5W0yJeS/Dht4HlUELrye2JZ +7TCQpx2Hs1FY5Tkj4VLnYHTPftS6cLYNx6hQbWZMhj5qmP9ccQ8rqdgdLB5RqCn3 +zo8wbKFZ8ygYt1yZyNOfJLNTBjIcC1BCKoZosA7MWHUOwRtt1ARVmldsNH3iio0l +wHjyKNYd0Kqw4uGEg6sulK69lw4G8YTnKtCt0G8vCpQHyQepolOMF7Q1NZEw02/U +E54qgaaC+ym+BQsqqF5iodmuIfLX+W0kKDee2YYhjuxNaFcPhE5j35LlGHCsrL0X +h4+2VZSYXuAO5aWpwX9jrrSFyCJLD/aYGMgdrwIDAQABAoIBAEZrF2UZCidLSJA5 +evwgM9I/kM4if3Wxd+Xv54vCn13cwECo+GhLC2ebueRJDkjZhSPe7LBlx2RZ9gNO +w0kPlZZYFx3AiKcmF0mHCExZyEE++EVv5pKdWwDIiu73fLYn6MqqvRA3X1zJp7yq +bP1MskLyjwAMr40IIgLXztDVbykiRC2Rw+o5cu7o3e0p0sFqJsjCUKtXZuzLePOk +gYYZ4FsmmVYh7pf244NEQao+fT19RtFL85E17yAHv+YD7qUBdbxoWIuAher9N/C0 +vOj4xYbNxbkS0+BTbygLAog5mFtNbAGysUZZ3YOYfKYgj9/u+aKwr2ZS2zIEeJj0 +eAiHtWECgYEA48dqxrR76JyukHid+XyI4Nqt+2EHEeDi23WTTT6lSZL1F3I2q7FF +DSHOA3hGw57GAMNQYCSzYxC4TBpZwJ7/8NdhA/kJg7tLOqcvZtS3Bu5bzLqLOCqL +E1tgh2LrpWjit2v+VSsQlf+QjG7QAEiWtya+AOfNWenILfxk2VNPP3MCgYEA0kOM +ym/EcgcSSihbFyyYO4UHZZ7rWiPRB+BtatJbEADMXMlwSAXvvVCpWSZBKBKjIE2y +ZM+kvv50QUd4ue7dKVEnqOy26XuAmuTE14smx1QyNonRvBV/HItJ0tKfMIZbXOpq +S2ESXkFybCzdOfzWOhx0PHjr40w8XUeSZi0LodUCgYAsC8bhD8uaKpozA7AAq41I +deEI6DVWxrb3mx/V4xRRSuKsGwDpaIkixfOxhhOhBlXhleM4BEDQGk6ZIMtUTSrO +5scy3nhxick9WVD4QI/3/iWwTC5ZuRhVsOjUpVNOFB8rOu3eiEpXxyirj04Xj/Hd +DtfVEv4JsgRsqA7UW6DKcwKBgQCiCvMXFDnWEwMSabWBz5lmzWfc9jO1HUM8Ccbp +e0I4vBTDMW854nFXejF5BhVS18Il5BsmvCvgEePwZy9wQ9jnvaaN9hglKkv7k3Ds +GE6DcazdASvFAuAaVHJJao7Ka9E/c10FyMLKJzASlCTOSr+iu0kNTbelTZx72uvF +mNONHQKBgCEUuJMM11mV0FCsVfJsmIv6z/zqOiPiOVbP1Bv2WlVzipvkI9bm6OyN +VHO8+oqFWyhJ3qRzebuPIefL8U6xjfMshX8MB23cB0J5LTPDZH3LCSmFvjr942EK +5+ewYHKtmS+6aaE+J+oB11r7XU8FyEI0kv6rAPDwJ19K4BMG/x7J +-----END RSA PRIVATE KEY-----` + +func TestKeyLoad_existing_pkcs1(t *testing.T) { + m := Modifier{} + m.log = testutils.Logger(t, m.Name()) + + dir := t.TempDir() + + if err := os.WriteFile(filepath.Join(dir, "testkey.key"), []byte(pkeyRSA), 0o600); err != nil { + t.Fatal(err) + } + + signer, newKey, err := m.loadOrGenerateKey(filepath.Join(dir, "testkey.key"), "rsa2048") + if err != nil { + t.Fatal(err) + } + if newKey { + t.Fatal("newKey=true") + } + + pubkey := signer.Public().(*rsa.PublicKey) + if pubkey.E != 65537 { + t.Fatalf("wrong public key returned by loadOrGenerateKey, got %d", pubkey.E) + } + if pubkey.N.String() != "23617257632228188386824425094266725423560758883229529475904285522114491665694237598874002862630696077162868821164059728985148713872807170386818903503533709975391952347175641552635505497204925274569104682448177717429244936284920784061388978739927939000424446717818401440783667723710780854637197555911253613285419663410256437304926940168312631109994734698918250930969511949067760562140706765511288141008942649676427142664185811322596443990204153105455693515405445788622172538582060141770589195075185467867938584021491237815987395835392935511032761463924045865609068314478096903374718657496007822964380498648030935260591" { + t.Fatalf("wrong public key returned by loadOrGenerateKey, got %s", pubkey.N.String()) + } +} diff --git a/internal/modify/group.go b/internal/modify/group.go new file mode 100644 index 0000000..3278b0f --- /dev/null +++ b/internal/modify/group.go @@ -0,0 +1,140 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package modify + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/module" +) + +type ( + // Group wraps multiple modifiers and runs them serially. + // + // It is also registered as a module under 'modifiers' name and acts as a + // module group. + Group struct { + instName string + Modifiers []module.Modifier + } + + groupState struct { + states []module.ModifierState + } +) + +func (g *Group) Init(cfg *config.Map) error { + for _, node := range cfg.Block.Children { + mod, err := modconfig.MsgModifier(cfg.Globals, append([]string{node.Name}, node.Args...), node) + if err != nil { + return err + } + + g.Modifiers = append(g.Modifiers, mod) + } + + return nil +} + +func (g *Group) Name() string { + return "modifiers" +} + +func (g *Group) InstanceName() string { + return g.instName +} + +func (g Group) ModStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.ModifierState, error) { + gs := groupState{} + for _, modifier := range g.Modifiers { + state, err := modifier.ModStateForMsg(ctx, msgMeta) + if err != nil { + // Free state objects we initialized already. + for _, state := range gs.states { + state.Close() + } + return nil, err + } + gs.states = append(gs.states, state) + } + return gs, nil +} + +func (gs groupState) RewriteSender(ctx context.Context, mailFrom string) (string, error) { + var err error + for _, state := range gs.states { + mailFrom, err = state.RewriteSender(ctx, mailFrom) + if err != nil { + return "", err + } + } + return mailFrom, nil +} + +func (gs groupState) RewriteRcpt(ctx context.Context, rcptTo string) ([]string, error) { + var err error + var result = []string{rcptTo} + for _, state := range gs.states { + var intermediateResult = []string{} + for _, partResult := range result { + var partResult_multi []string + partResult_multi, err = state.RewriteRcpt(ctx, partResult) + if err != nil { + return []string{""}, err + } + intermediateResult = append(intermediateResult, partResult_multi...) + } + result = intermediateResult + } + return result, nil +} + +func (gs groupState) RewriteBody(ctx context.Context, h *textproto.Header, body buffer.Buffer) error { + for _, state := range gs.states { + if err := state.RewriteBody(ctx, h, body); err != nil { + return err + } + } + return nil +} + +func (gs groupState) Close() error { + // We still try close all state objects to minimize + // resource leaks when Close fails for one object.. + + var lastErr error + for _, state := range gs.states { + if err := state.Close(); err != nil { + lastErr = err + } + } + return lastErr +} + +func init() { + module.Register("modifiers", func(_, instName string, _, _ []string) (module.Module, error) { + return &Group{ + instName: instName, + }, nil + }) +} diff --git a/internal/modify/replace_addr.go b/internal/modify/replace_addr.go new file mode 100644 index 0000000..50dd5df --- /dev/null +++ b/internal/modify/replace_addr.go @@ -0,0 +1,156 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package modify + +import ( + "context" + "fmt" + "strings" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/module" +) + +// replaceAddr is a simple module that replaces matching sender (or recipient) address +// in messages using module.Table implementation. +// +// If created with modName = "modify.replace_sender", it will change sender address. +// If created with modName = "modify.replace_rcpt", it will change recipient addresses. +type replaceAddr struct { + modName string + instName string + inlineArgs []string + + replaceSender bool + replaceRcpt bool + table module.MultiTable +} + +func NewReplaceAddr(modName, instName string, _, inlineArgs []string) (module.Module, error) { + r := replaceAddr{ + modName: modName, + instName: instName, + inlineArgs: inlineArgs, + replaceSender: modName == "modify.replace_sender", + replaceRcpt: modName == "modify.replace_rcpt", + } + + return &r, nil +} + +func (r *replaceAddr) Init(cfg *config.Map) error { + return modconfig.ModuleFromNode("table", r.inlineArgs, cfg.Block, cfg.Globals, &r.table) +} + +func (r replaceAddr) Name() string { + return r.modName +} + +func (r replaceAddr) InstanceName() string { + return r.instName +} + +func (r replaceAddr) ModStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.ModifierState, error) { + return r, nil +} + +func (r replaceAddr) RewriteSender(ctx context.Context, mailFrom string) (string, error) { + if r.replaceSender { + results, err := r.rewrite(ctx, mailFrom) + if err != nil { + return mailFrom, err + } + mailFrom = results[0] + } + return mailFrom, nil +} + +func (r replaceAddr) RewriteRcpt(ctx context.Context, rcptTo string) ([]string, error) { + if r.replaceRcpt { + return r.rewrite(ctx, rcptTo) + } + return []string{rcptTo}, nil +} + +func (r replaceAddr) RewriteBody(ctx context.Context, h *textproto.Header, body buffer.Buffer) error { + return nil +} + +func (r replaceAddr) Close() error { + return nil +} + +func (r replaceAddr) rewrite(ctx context.Context, val string) ([]string, error) { + normAddr, err := address.ForLookup(val) + if err != nil { + return []string{val}, fmt.Errorf("malformed address: %v", err) + } + + replacements, err := r.table.LookupMulti(ctx, normAddr) + if err != nil { + return []string{val}, err + } + if len(replacements) > 0 { + for _, replacement := range replacements { + if !address.Valid(replacement) { + return []string{""}, fmt.Errorf("refusing to replace recipient with the invalid address %s", replacement) + } + } + return replacements, nil + } + + mbox, domain, err := address.Split(normAddr) + if err != nil { + // If we have malformed address here, something is really wrong, but let's + // ignore it silently then anyway. + return []string{val}, nil + } + + // mbox is already normalized, since it is a part of address.ForLookup + // result. + replacements, err = r.table.LookupMulti(ctx, mbox) + if err != nil { + return []string{val}, err + } + if len(replacements) > 0 { + var results = make([]string, len(replacements)) + for i, replacement := range replacements { + if strings.Contains(replacement, "@") && !strings.HasPrefix(replacement, `"`) && !strings.HasSuffix(replacement, `"`) { + if !address.Valid(replacement) { + return []string{""}, fmt.Errorf("refusing to replace recipient with invalid address %s", replacement) + } + results[i] = replacement + } else { + results[i] = replacement + "@" + domain + } + } + return results, nil + } + + return []string{val}, nil +} + +func init() { + module.Register("modify.replace_sender", NewReplaceAddr) + module.Register("modify.replace_rcpt", NewReplaceAddr) +} diff --git a/internal/modify/replace_addr_test.go b/internal/modify/replace_addr_test.go new file mode 100644 index 0000000..e16cfd3 --- /dev/null +++ b/internal/modify/replace_addr_test.go @@ -0,0 +1,114 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package modify + +import ( + "context" + "reflect" + "testing" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/internal/testutils" +) + +func testReplaceAddr(t *testing.T, modName string) { + test := func(addr string, expectedMulti []string, aliases map[string][]string) { + t.Helper() + + mod, err := NewReplaceAddr(modName, "", nil, []string{"dummy"}) + if err != nil { + t.Fatal(err) + } + m := mod.(*replaceAddr) + if err := m.Init(config.NewMap(nil, config.Node{})); err != nil { + t.Fatal(err) + } + m.table = testutils.MultiTable{M: aliases} + + var actualMulti []string + if modName == "modify.replace_sender" { + var actual string + actual, err = m.RewriteSender(context.Background(), addr) + if err != nil { + t.Fatal(err) + } + actualMulti = []string{actual} + } + if modName == "modify.replace_rcpt" { + actualMulti, err = m.RewriteRcpt(context.Background(), addr) + if err != nil { + t.Fatal(err) + } + } + + if !reflect.DeepEqual(actualMulti, expectedMulti) { + t.Errorf("want %s, got %s", expectedMulti, actualMulti) + } + } + + test("test@example.org", []string{"test@example.org"}, nil) + test("postmaster", []string{"postmaster"}, nil) + test("test@example.com", []string{"test@example.org"}, + map[string][]string{"test@example.com": []string{"test@example.org"}}) + test(`"\"test @ test\""@example.com`, []string{"test@example.org"}, + map[string][]string{`"\"test @ test\""@example.com`: []string{"test@example.org"}}) + test(`test@example.com`, []string{`"\"test @ test\""@example.org`}, + map[string][]string{`test@example.com`: []string{`"\"test @ test\""@example.org`}}) + test(`"\"test @ test\""@example.com`, []string{`"\"b @ b\""@example.com`}, + map[string][]string{`"\"test @ test\""`: []string{`"\"b @ b\""`}}) + test("TeSt@eXAMple.com", []string{"test@example.org"}, + map[string][]string{"test@example.com": []string{"test@example.org"}}) + test("test@example.com", []string{"test2@example.com"}, + map[string][]string{"test": []string{"test2"}}) + test("test@example.com", []string{"test2@example.org"}, + map[string][]string{"test": []string{"test2@example.org"}}) + test("postmaster", []string{"test2@example.org"}, + map[string][]string{"postmaster": []string{"test2@example.org"}}) + test("TeSt@examPLE.com", []string{"test2@example.com"}, + map[string][]string{"test": []string{"test2"}}) + test("test@example.com", []string{"test3@example.com"}, + map[string][]string{ + "test@example.com": []string{"test3@example.com"}, + "test": []string{"test2"}, + }) + test("rcpt@E\u0301.example.com", []string{"rcpt@foo.example.com"}, + map[string][]string{ + "rcpt@\u00E9.example.com": []string{"rcpt@foo.example.com"}, + }) + test("E\u0301@foo.example.com", []string{"rcpt@foo.example.com"}, + map[string][]string{ + "\u00E9@foo.example.com": []string{"rcpt@foo.example.com"}, + }) + + if modName == "modify.replace_rcpt" { + //multiple aliases + test("test@example.com", []string{"test@example.org", "test@example.net"}, + map[string][]string{"test@example.com": []string{"test@example.org", "test@example.net"}}) + test("test@example.com", []string{"1@example.com", "2@example.com", "3@example.com"}, + map[string][]string{"test@example.com": []string{"1@example.com", "2@example.com", "3@example.com"}}) + } +} + +func TestReplaceAddr_RewriteSender(t *testing.T) { + testReplaceAddr(t, "modify.replace_sender") +} + +func TestReplaceAddr_RewriteRcpt(t *testing.T) { + testReplaceAddr(t, "modify.replace_rcpt") +} diff --git a/internal/msgpipeline/bench_test.go b/internal/msgpipeline/bench_test.go new file mode 100644 index 0000000..ae9aa8e --- /dev/null +++ b/internal/msgpipeline/bench_test.go @@ -0,0 +1,98 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "strconv" + "testing" + + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func BenchmarkMsgPipelineSimple(b *testing.B) { + target := testutils.Target{InstName: "test_target", DiscardMessages: true} + d := MsgPipeline{msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }} + + testutils.BenchDelivery(b, &d, "sender@example.org", []string{"rcpt-X@example.org"}) +} + +func BenchmarkMsgPipelineGlobalChecks(b *testing.B) { + testWithCount := func(checksCount int) { + b.Run(strconv.Itoa(checksCount), func(b *testing.B) { + checks := make([]module.Check, 0, checksCount) + for i := 0; i < checksCount; i++ { + checks = append(checks, &testutils.Check{InstName: "check_" + strconv.Itoa(i)}) + } + + target := testutils.Target{InstName: "test_target", DiscardMessages: true} + d := MsgPipeline{msgpipelineCfg: msgpipelineCfg{ + globalChecks: checks, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }} + + testutils.BenchDelivery(b, &d, "sender@example.org", []string{"rcpt-X@example.org"}) + }) + } + + testWithCount(5) + testWithCount(10) + testWithCount(15) +} + +func BenchmarkMsgPipelineTargets(b *testing.B) { + testWithCount := func(targetCount int) { + b.Run(strconv.Itoa(targetCount), func(b *testing.B) { + targets := make([]module.DeliveryTarget, 0, targetCount) + for i := 0; i < targetCount; i++ { + targets = append(targets, &testutils.Target{InstName: "target_" + strconv.Itoa(i), DiscardMessages: true}) + } + + d := MsgPipeline{msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: targets, + }, + }, + }} + + testutils.BenchDelivery(b, &d, "sender@example.org", []string{"rcpt-X@example.org"}) + }) + } + + testWithCount(5) + testWithCount(10) + testWithCount(15) +} diff --git a/internal/msgpipeline/bodynonatomic_test.go b/internal/msgpipeline/bodynonatomic_test.go new file mode 100644 index 0000000..c441980 --- /dev/null +++ b/internal/msgpipeline/bodynonatomic_test.go @@ -0,0 +1,148 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "errors" + "testing" + + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/modify" + "github.com/foxcpp/maddy/internal/testutils" +) + +type multipleErrs map[string]error + +func (m multipleErrs) SetStatus(rcptTo string, err error) { + m[rcptTo] = err +} + +func TestMsgPipeline_BodyNonAtomic(t *testing.T) { + err := errors.New("go away") + + target := testutils.Target{ + PartialBodyErr: map[string]error{ + "tester@example.org": err, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + c := multipleErrs{} + testutils.DoTestDeliveryNonAtomic(t, c, &d, "sender@example.org", []string{"tester@example.org", "tester2@example.org"}) + + if c["tester@example.org"] == nil { + t.Fatalf("no error for tester@example.org") + } + if c["tester@example.org"].Error() != err.Error() { + t.Errorf("wrong error for tester@example.org: %v", err) + } +} + +func TestMsgPipeline_BodyNonAtomic_ModifiedRcpt(t *testing.T) { + err := errors.New("go away") + + target := testutils.Target{ + PartialBodyErr: map[string]error{ + "tester-alias@example.org": err, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{ + testutils.Modifier{ + InstName: "test_modifier", + RcptTo: map[string][]string{ + "tester@example.org": []string{"tester-alias@example.org"}, + }, + }, + }, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + c := multipleErrs{} + testutils.DoTestDeliveryNonAtomic(t, c, &d, "sender@example.org", []string{"tester@example.org"}) + + if c["tester@example.org"] == nil { + t.Fatalf("no error for tester@example.org") + } + if c["tester@example.org"].Error() != err.Error() { + t.Errorf("wrong error for tester@example.org: %v", err) + } +} + +func TestMsgPipeline_BodyNonAtomic_ExpandAtomic(t *testing.T) { + err := errors.New("go away") + + target, target2 := testutils.Target{ + PartialBodyErr: map[string]error{ + "tester@example.org": err, + }, + }, testutils.Target{ + BodyErr: err, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target, &target2}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + c := multipleErrs{} + testutils.DoTestDeliveryNonAtomic(t, c, &d, "sender@example.org", []string{"tester@example.org", "tester2@example.org"}) + + if c["tester@example.org"] == nil { + t.Fatalf("no error for tester@example.org") + } + if c["tester@example.org"].Error() != err.Error() { + t.Errorf("wrong error for tester@example.org: %v", err) + } + if c["tester2@example.org"] == nil { + t.Fatalf("no error for tester@example.org") + } + if c["tester2@example.org"].Error() != err.Error() { + t.Errorf("wrong error for tester@example.org: %v", err) + } +} diff --git a/internal/msgpipeline/check_group.go b/internal/msgpipeline/check_group.go new file mode 100644 index 0000000..1fd6b24 --- /dev/null +++ b/internal/msgpipeline/check_group.go @@ -0,0 +1,67 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/module" +) + +// CheckGroup is a module container for a group of Check implementations. +// +// It allows to share a set of filter configurations between using named +// configuration blocks (module instances) system. +// +// It is registered globally under the name 'checks'. The object does not +// implement any standard module interfaces besides module.Module and is +// specific to the message pipeline. +type CheckGroup struct { + instName string + L []module.Check +} + +func (cg *CheckGroup) Init(cfg *config.Map) error { + for _, node := range cfg.Block.Children { + chk, err := modconfig.MessageCheck(cfg.Globals, append([]string{node.Name}, node.Args...), node) + if err != nil { + return err + } + + cg.L = append(cg.L, chk) + } + + return nil +} + +func (CheckGroup) Name() string { + return "checks" +} + +func (cg CheckGroup) InstanceName() string { + return cg.instName +} + +func init() { + module.Register("checks", func(_, instName string, _, _ []string) (module.Module, error) { + return &CheckGroup{ + instName: instName, + }, nil + }) +} diff --git a/internal/msgpipeline/check_runner.go b/internal/msgpipeline/check_runner.go new file mode 100644 index 0000000..f6df952 --- /dev/null +++ b/internal/msgpipeline/check_runner.go @@ -0,0 +1,349 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "context" + "runtime/debug" + "sync" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/dmarc" +) + +// checkRunner runs groups of checks, collects and merges results. +// It also makes sure that each check gets only one state object created. +type checkRunner struct { + msgMeta *module.MsgMetadata + mailFrom string + mailFromReceived bool + + checkedRcpts []string + checkedRcptsPerCheck map[module.CheckState]map[string]struct{} + checkedRcptsLock sync.Mutex + + resolver dns.Resolver + doDMARC bool + didDMARCFetch bool + dmarcVerify *dmarc.Verifier + + log log.Logger + + states map[module.Check]module.CheckState + + mergedRes module.CheckResult +} + +func newCheckRunner(msgMeta *module.MsgMetadata, log log.Logger, r dns.Resolver) *checkRunner { + return &checkRunner{ + msgMeta: msgMeta, + checkedRcptsPerCheck: map[module.CheckState]map[string]struct{}{}, + log: log, + resolver: r, + dmarcVerify: dmarc.NewVerifier(r), + states: make(map[module.Check]module.CheckState), + } +} + +func (cr *checkRunner) checkStates(ctx context.Context, checks []module.Check) ([]module.CheckState, error) { + states := make([]module.CheckState, 0, len(checks)) + newStates := make([]module.CheckState, 0, len(checks)) + newStatesMap := make(map[module.Check]module.CheckState, len(checks)) + closeStates := func() { + for _, state := range states { + state.Close() + } + } + + for _, check := range checks { + state, ok := cr.states[check] + if ok { + states = append(states, state) + continue + } + + cr.log.Debugf("initializing state for %v (%p)", objectName(check), check) + state, err := check.CheckStateForMsg(ctx, cr.msgMeta) + if err != nil { + closeStates() + return nil, err + } + states = append(states, state) + newStates = append(newStates, state) + newStatesMap[check] = state + } + + if len(newStates) == 0 { + return states, nil + } + + // Here we replay previous CheckConnection/CheckSender/CheckRcpt calls + // for any newly initialized checks so they all get change to see all these things. + // + // Done outside of check loop above to make sure we can run these for multiple + // checks in parallel. + if cr.mailFromReceived { + err := cr.runAndMergeResults(newStates, func(s module.CheckState) module.CheckResult { + res := s.CheckConnection(ctx) + return res + }) + if err != nil { + closeStates() + return nil, err + } + err = cr.runAndMergeResults(newStates, func(s module.CheckState) module.CheckResult { + res := s.CheckSender(ctx, cr.mailFrom) + return res + }) + if err != nil { + closeStates() + return nil, err + } + } + + if len(cr.checkedRcpts) != 0 { + for _, rcpt := range cr.checkedRcpts { + err := cr.runAndMergeResults(states, func(s module.CheckState) module.CheckResult { + // Avoid calling CheckRcpt for the same recipient for the same check + // multiple times, even if requested. + cr.checkedRcptsLock.Lock() + if _, ok := cr.checkedRcptsPerCheck[s][rcpt]; ok { + cr.checkedRcptsLock.Unlock() + return module.CheckResult{} + } + if cr.checkedRcptsPerCheck[s] == nil { + cr.checkedRcptsPerCheck[s] = make(map[string]struct{}) + } + cr.checkedRcptsPerCheck[s][rcpt] = struct{}{} + cr.checkedRcptsLock.Unlock() + + res := s.CheckRcpt(ctx, rcpt) + return res + }) + if err != nil { + closeStates() + return nil, err + } + } + } + + // This is done after all actions that can fail so we will not have to remove + // state objects from main map. + for check, state := range newStatesMap { + cr.states[check] = state + } + + return states, nil +} + +func (cr *checkRunner) runAndMergeResults(states []module.CheckState, runner func(module.CheckState) module.CheckResult) error { + data := struct { + authResLock sync.Mutex + headerLock sync.Mutex + + quarantineErr error + quarantineCheck string + setQuarantineErr sync.Once + + rejectErr error + rejectCheck string + setRejectErr sync.Once + + wg sync.WaitGroup + }{} + + for _, state := range states { + data.wg.Add(1) + go func() { + defer func() { + data.wg.Done() + if err := recover(); err != nil { + stack := debug.Stack() + log.Printf("panic during check execution: %v\n%s", err, stack) + } + }() + + subCheckRes := runner(state) + + // We check the length because we don't want to take locks + // when it is not necessary. + if len(subCheckRes.AuthResult) != 0 { + data.authResLock.Lock() + cr.mergedRes.AuthResult = append(cr.mergedRes.AuthResult, subCheckRes.AuthResult...) + data.authResLock.Unlock() + } + if subCheckRes.Header.Len() != 0 { + data.headerLock.Lock() + for field := subCheckRes.Header.Fields(); field.Next(); { + formatted, err := field.Raw() + if err != nil { + cr.log.Error("malformed header field added by check", err) + } + cr.mergedRes.Header.AddRaw(formatted) + } + data.headerLock.Unlock() + } + + if subCheckRes.Quarantine { + data.setQuarantineErr.Do(func() { + data.quarantineErr = subCheckRes.Reason + }) + } else if subCheckRes.Reject { + data.setRejectErr.Do(func() { + data.rejectErr = subCheckRes.Reason + }) + } else if subCheckRes.Reason != nil { + // 'action ignore' case. There is Reason, but action.Apply set + // both Reject and Quarantine to false. Log the reason for + // purposes of deployment testing. + cr.log.Error("no check action", subCheckRes.Reason) + } + }() + } + + data.wg.Wait() + if data.rejectErr != nil { + return data.rejectErr + } + + if data.quarantineErr != nil { + cr.log.Error("quarantined", data.quarantineErr) + cr.mergedRes.Quarantine = true + } + + return nil +} + +func (cr *checkRunner) checkConnSender(ctx context.Context, checks []module.Check, mailFrom string) error { + cr.mailFrom = mailFrom + cr.mailFromReceived = true + + // checkStates will run CheckConnection and CheckSender. + _, err := cr.checkStates(ctx, checks) + return err +} + +func (cr *checkRunner) checkRcpt(ctx context.Context, checks []module.Check, rcptTo string) error { + states, err := cr.checkStates(ctx, checks) + if err != nil { + return err + } + + err = cr.runAndMergeResults(states, func(s module.CheckState) module.CheckResult { + cr.checkedRcptsLock.Lock() + if _, ok := cr.checkedRcptsPerCheck[s][rcptTo]; ok { + cr.checkedRcptsLock.Unlock() + return module.CheckResult{} + } + if cr.checkedRcptsPerCheck[s] == nil { + cr.checkedRcptsPerCheck[s] = make(map[string]struct{}) + } + cr.checkedRcptsPerCheck[s][rcptTo] = struct{}{} + cr.checkedRcptsLock.Unlock() + + res := s.CheckRcpt(ctx, rcptTo) + return res + }) + + cr.checkedRcpts = append(cr.checkedRcpts, rcptTo) + return err +} + +func (cr *checkRunner) checkBody(ctx context.Context, checks []module.Check, header textproto.Header, body buffer.Buffer) error { + states, err := cr.checkStates(ctx, checks) + if err != nil { + return err + } + + if cr.doDMARC && !cr.didDMARCFetch { + cr.dmarcVerify.FetchRecord(ctx, header) + cr.didDMARCFetch = true + } + + return cr.runAndMergeResults(states, func(s module.CheckState) module.CheckResult { + res := s.CheckBody(ctx, header, body) + return res + }) +} + +func (cr *checkRunner) applyResults(hostname string, header *textproto.Header) error { + if cr.mergedRes.Quarantine { + cr.msgMeta.Quarantine = true + } + + if cr.doDMARC { + dmarcRes, policy := cr.dmarcVerify.Apply(cr.mergedRes.AuthResult) + cr.mergedRes.AuthResult = append(cr.mergedRes.AuthResult, &dmarcRes.Authres) + switch policy { + case dmarc.PolicyReject: + code := 550 + enchCode := exterrors.EnhancedCode{5, 7, 1} + if dmarcRes.Authres.Value == authres.ResultTempError { + code = 450 + enchCode[0] = 4 + } + return &exterrors.SMTPError{ + Code: code, + EnhancedCode: enchCode, + Message: "DMARC check failed", + CheckName: "dmarc", + Misc: map[string]interface{}{ + "reason": dmarcRes.Authres.Reason, + "dkim_res": dmarcRes.DKIMResult.Value, + "dkim_domain": dmarcRes.DKIMResult.Domain, + "spf_res": dmarcRes.SPFResult.Value, + "spf_from": dmarcRes.SPFResult.From, + }, + } + case dmarc.PolicyQuarantine: + cr.msgMeta.Quarantine = true + + // Mimick the message structure for regular checks. + cr.log.Msg("quarantined", "reason", dmarcRes.Authres.Reason, "check", "dmarc") + } + } + + // After results for all checks are checked, authRes will be populated with values + // we should put into Authentication-Results header. + if len(cr.mergedRes.AuthResult) != 0 { + header.Add("Authentication-Results", authres.Format(hostname, cr.mergedRes.AuthResult)) + } + + for field := cr.mergedRes.Header.Fields(); field.Next(); { + formatted, err := field.Raw() + if err != nil { + cr.log.Error("malformed header field added by check", err) + } + header.AddRaw(formatted) + } + return nil +} + +func (cr *checkRunner) close() { + cr.dmarcVerify.Close() + for _, state := range cr.states { + state.Close() + } +} diff --git a/internal/msgpipeline/check_test.go b/internal/msgpipeline/check_test.go new file mode 100644 index 0000000..72fcb79 --- /dev/null +++ b/internal/msgpipeline/check_test.go @@ -0,0 +1,448 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "errors" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestMsgPipeline_Checks(t *testing.T) { + target := testutils.Target{} + check1, check2 := testutils.Check{}, testutils.Check{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&check1, &check2}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "whatever@whatever", []string{"whatever@whatever"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + if target.Messages[0].MsgMeta.Quarantine { + t.Fatalf("message is quarantined when it shouldn't") + } + + if check1.UnclosedStates != 0 || check2.UnclosedStates != 0 { + t.Fatalf("checks state objects leak or double-closed, alive counters: %v, %v", check1.UnclosedStates, check2.UnclosedStates) + } +} + +func TestMsgPipeline_AuthResults(t *testing.T) { + target := testutils.Target{} + check1, check2 := testutils.Check{ + BodyRes: module.CheckResult{ + AuthResult: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultFail, + From: "FROM", + Helo: "HELO", + }, + }, + }, + }, testutils.Check{ + BodyRes: module.CheckResult{ + AuthResult: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultFail, + From: "FROM2", + Helo: "HELO2", + }, + }, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&check1, &check2}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Hostname: "TEST-HOST", + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "whatever@whatever", []string{"whatever@whatever"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + authRes := target.Messages[0].Header.Get("Authentication-Results") + id, parsed, err := authres.Parse(authRes) + if err != nil { + t.Fatalf("failed to parse results") + } + if id != "TEST-HOST" { + t.Fatalf("wrong authres identifier") + } + if len(parsed) != 2 { + t.Fatalf("wrong amount of parts, want %d, got %d", 2, len(parsed)) + } + + var seen1, seen2 bool + for _, parts := range parsed { + spfPart, ok := parts.(*authres.SPFResult) + if !ok { + t.Fatalf("Not SPFResult") + } + + if spfPart.From == "FROM" { + seen1 = true + } + if spfPart.From == "FROM2" { + seen2 = true + } + } + + if !seen1 { + t.Fatalf("First authRes is missing") + } + if !seen2 { + t.Fatalf("Second authRes is missing") + } + + if check1.UnclosedStates != 0 || check2.UnclosedStates != 0 { + t.Fatalf("checks state objects leak or double-closed, alive counters: %v, %v", check1.UnclosedStates, check2.UnclosedStates) + } +} + +func TestMsgPipeline_Headers(t *testing.T) { + hdr1 := textproto.Header{} + hdr1.Add("HDR1", "1") + hdr2 := textproto.Header{} + hdr2.Add("HDR2", "2") + + target := testutils.Target{} + check1, check2 := testutils.Check{ + BodyRes: module.CheckResult{ + Header: hdr1, + }, + }, testutils.Check{ + BodyRes: module.CheckResult{ + Header: hdr2, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&check1, &check2}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Hostname: "TEST-HOST", + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "whatever@whatever", []string{"whatever@whatever"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + if target.Messages[0].Header.Get("HDR1") != "1" { + t.Fatalf("wrong HDR1 value, want %s, got %s", "1", target.Messages[0].Header.Get("HDR1")) + } + if target.Messages[0].Header.Get("HDR2") != "2" { + t.Fatalf("wrong HDR2 value, want %s, got %s", "1", target.Messages[0].Header.Get("HDR2")) + } + + if check1.UnclosedStates != 0 || check2.UnclosedStates != 0 { + t.Fatalf("checks state objects leak or double-closed, alive counters: %v, %v", check1.UnclosedStates, check2.UnclosedStates) + } +} + +func TestMsgPipeline_Globalcheck_Errors(t *testing.T) { + target := testutils.Target{} + check_ := testutils.Check{ + InitErr: errors.New("1"), + ConnRes: module.CheckResult{Reject: true, Reason: errors.New("2")}, + SenderRes: module.CheckResult{Reject: true, Reason: errors.New("3")}, + RcptRes: module.CheckResult{Reject: true, Reason: errors.New("4")}, + BodyRes: module.CheckResult{Reject: true, Reason: errors.New("5")}, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&check_}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Hostname: "TEST-HOST", + Log: testutils.Logger(t, "msgpipeline"), + } + + t.Run("init err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.InitErr = nil + + t.Run("conn err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.ConnRes.Reject = false + + t.Run("mail from err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.SenderRes.Reject = false + + t.Run("rcpt to err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.RcptRes.Reject = false + + t.Run("body err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.BodyRes.Reject = false + + t.Run("no err", func(t *testing.T) { + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + }) + + if check_.UnclosedStates != 0 { + t.Fatalf("check state objects leak or double-closed, counters: %d", check_.UnclosedStates) + } +} + +func TestMsgPipeline_SourceCheck_Errors(t *testing.T) { + target := testutils.Target{} + check_ := testutils.Check{ + InitErr: errors.New("1"), + ConnRes: module.CheckResult{Reject: true, Reason: errors.New("2")}, + SenderRes: module.CheckResult{Reject: true, Reason: errors.New("3")}, + RcptRes: module.CheckResult{Reject: true, Reason: errors.New("4")}, + BodyRes: module.CheckResult{Reject: true, Reason: errors.New("5")}, + } + globalCheck := testutils.Check{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&globalCheck}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + checks: []module.Check{&check_}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Hostname: "TEST-HOST", + Log: testutils.Logger(t, "msgpipeline"), + } + + t.Run("init err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.InitErr = nil + + t.Run("conn err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.ConnRes.Reject = false + + t.Run("mail from err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.SenderRes.Reject = false + + t.Run("rcpt to err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.RcptRes.Reject = false + + t.Run("body err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.BodyRes.Reject = false + + t.Run("no err", func(t *testing.T) { + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + }) + + if check_.UnclosedStates != 0 || globalCheck.UnclosedStates != 0 { + t.Fatalf("check state objects leak or double-closed, counters: %d, %d", + check_.UnclosedStates, globalCheck.UnclosedStates) + } +} + +func TestMsgPipeline_RcptCheck_Errors(t *testing.T) { + target := testutils.Target{} + check_ := testutils.Check{ + InitErr: errors.New("1"), + ConnRes: module.CheckResult{Reject: true, Reason: errors.New("2")}, + SenderRes: module.CheckResult{Reject: true, Reason: errors.New("3")}, + RcptRes: module.CheckResult{Reject: true, Reason: errors.New("4")}, + BodyRes: module.CheckResult{Reject: true, Reason: errors.New("5")}, + + InstName: "err_check", + } + // Added to check whether it leaks. + globalCheck := testutils.Check{InstName: "global_check"} + sourceCheck := testutils.Check{InstName: "source_check"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&globalCheck}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + checks: []module.Check{&check_}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Hostname: "TEST-HOST", + Log: testutils.Logger(t, "msgpipeline"), + } + + t.Run("init err", func(t *testing.T) { + d.Log = testutils.Logger(t, "msgpipeline") + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + + t.Log("!!!", check_.UnclosedStates) + }) + + check_.InitErr = nil + + t.Run("conn err", func(t *testing.T) { + d.Log = testutils.Logger(t, "msgpipeline") + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + + t.Log("!!!", check_.UnclosedStates) + }) + + check_.ConnRes.Reject = false + + t.Run("mail from err", func(t *testing.T) { + d.Log = testutils.Logger(t, "msgpipeline") + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + + t.Log("!!!", check_.UnclosedStates) + }) + + check_.SenderRes.Reject = false + + t.Run("rcpt to err", func(t *testing.T) { + d.Log = testutils.Logger(t, "msgpipeline") + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.RcptRes.Reject = false + + t.Run("body err", func(t *testing.T) { + d.Log = testutils.Logger(t, "msgpipeline") + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.BodyRes.Reject = false + + t.Run("no err", func(t *testing.T) { + d.Log = testutils.Logger(t, "msgpipeline") + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + }) + + if check_.UnclosedStates != 0 || sourceCheck.UnclosedStates != 0 || globalCheck.UnclosedStates != 0 { + t.Fatalf("check state objects leak or double-closed, counters: %d, %d, %d", + check_.UnclosedStates, sourceCheck.UnclosedStates, globalCheck.UnclosedStates) + } +} diff --git a/internal/msgpipeline/config.go b/internal/msgpipeline/config.go new file mode 100644 index 0000000..46693e6 --- /dev/null +++ b/internal/msgpipeline/config.go @@ -0,0 +1,397 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "fmt" + "strconv" + "strings" + + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/modify" +) + +type sourceIn struct { + t module.Table + block sourceBlock +} + +type msgpipelineCfg struct { + globalChecks []module.Check + globalModifiers modify.Group + sourceIn []sourceIn + perSource map[string]sourceBlock + defaultSource sourceBlock + doDMARC bool +} + +func parseMsgPipelineRootCfg(globals map[string]interface{}, nodes []config.Node) (msgpipelineCfg, error) { + cfg := msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + } + var defaultSrcRaw []config.Node + var othersRaw []config.Node + for _, node := range nodes { + switch node.Name { + case "check": + globalChecks, err := parseChecksGroup(globals, node) + if err != nil { + return msgpipelineCfg{}, err + } + + cfg.globalChecks = append(cfg.globalChecks, globalChecks...) + case "modify": + globalModifiers, err := parseModifiersGroup(globals, node) + if err != nil { + return msgpipelineCfg{}, err + } + + cfg.globalModifiers.Modifiers = append(cfg.globalModifiers.Modifiers, globalModifiers.Modifiers...) + case "source_in": + var tbl module.Table + if err := modconfig.ModuleFromNode("table", node.Args, config.Node{}, globals, &tbl); err != nil { + return msgpipelineCfg{}, err + } + srcBlock, err := parseMsgPipelineSrcCfg(globals, node.Children) + if err != nil { + return msgpipelineCfg{}, err + } + cfg.sourceIn = append(cfg.sourceIn, sourceIn{ + t: tbl, + block: srcBlock, + }) + case "source": + srcBlock, err := parseMsgPipelineSrcCfg(globals, node.Children) + if err != nil { + return msgpipelineCfg{}, err + } + + if len(node.Args) == 0 { + return msgpipelineCfg{}, config.NodeErr(node, "expected at least one source matching rule") + } + + for _, rule := range node.Args { + if strings.Contains(rule, "@") { + rule, err = address.ForLookup(rule) + } else { + rule, err = dns.ForLookup(rule) + } + if err != nil { + return msgpipelineCfg{}, config.NodeErr(node, "invalid source match rule: %v: %v", rule, err) + } + + if !validMatchRule(rule) { + return msgpipelineCfg{}, config.NodeErr(node, "invalid source routing rule: %v", rule) + } + + if _, ok := cfg.perSource[rule]; ok { + continue + } + + cfg.perSource[rule] = srcBlock + } + case "default_source": + if defaultSrcRaw != nil { + return msgpipelineCfg{}, config.NodeErr(node, "duplicate 'default_source' block") + } + defaultSrcRaw = node.Children + case "dmarc": + switch len(node.Args) { + case 1: + switch node.Args[0] { + case "yes": + cfg.doDMARC = true + case "no": + default: + return msgpipelineCfg{}, config.NodeErr(node, "invalid argument for dmarc") + } + case 0: + cfg.doDMARC = true + } + case "deliver_to", "reroute", "destination_in", "destination", "default_destination", "reject": + othersRaw = append(othersRaw, node) + default: + return msgpipelineCfg{}, config.NodeErr(node, "unknown pipeline directive: %s", node.Name) + } + } + + if len(cfg.perSource) == 0 && len(defaultSrcRaw) == 0 { + if len(othersRaw) == 0 { + return msgpipelineCfg{}, fmt.Errorf("empty pipeline configuration, use 'reject' to reject messages") + } + + var err error + cfg.defaultSource, err = parseMsgPipelineSrcCfg(globals, othersRaw) + return cfg, err + } else if len(othersRaw) != 0 { + return msgpipelineCfg{}, config.NodeErr(othersRaw[0], "can't put handling directives together with source rules, did you mean to put it into 'default_source' block or into all source blocks?") + } + + if len(defaultSrcRaw) == 0 { + return msgpipelineCfg{}, config.NodeErr(nodes[0], "missing or empty default source block, use default_source { reject } to reject messages") + } + + var err error + cfg.defaultSource, err = parseMsgPipelineSrcCfg(globals, defaultSrcRaw) + return cfg, err +} + +func parseMsgPipelineSrcCfg(globals map[string]interface{}, nodes []config.Node) (sourceBlock, error) { + src := sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + } + var defaultRcptRaw []config.Node + var othersRaw []config.Node + for _, node := range nodes { + switch node.Name { + case "check": + checks, err := parseChecksGroup(globals, node) + if err != nil { + return sourceBlock{}, err + } + + src.checks = append(src.checks, checks...) + case "modify": + modifiers, err := parseModifiersGroup(globals, node) + if err != nil { + return sourceBlock{}, err + } + + src.modifiers.Modifiers = append(src.modifiers.Modifiers, modifiers.Modifiers...) + case "destination_in": + var tbl module.Table + if err := modconfig.ModuleFromNode("table", node.Args, config.Node{}, globals, &tbl); err != nil { + return sourceBlock{}, err + } + rcptBlock, err := parseMsgPipelineRcptCfg(globals, node.Children) + if err != nil { + return sourceBlock{}, err + } + src.rcptIn = append(src.rcptIn, rcptIn{ + t: tbl, + block: rcptBlock, + }) + case "destination": + rcptBlock, err := parseMsgPipelineRcptCfg(globals, node.Children) + if err != nil { + return sourceBlock{}, err + } + + if len(node.Args) == 0 { + return sourceBlock{}, config.NodeErr(node, "expected at least one destination match rule") + } + + for _, rule := range node.Args { + if strings.Contains(rule, "@") { + rule, err = address.ForLookup(rule) + } else { + rule, err = dns.ForLookup(rule) + } + if err != nil { + return sourceBlock{}, config.NodeErr(node, "invalid destination match rule: %v: %v", rule, err) + } + + if !validMatchRule(rule) { + return sourceBlock{}, config.NodeErr(node, "invalid destination match rule: %v", rule) + } + + if _, ok := src.perRcpt[rule]; ok { + continue + } + + src.perRcpt[rule] = rcptBlock + } + case "default_destination": + if defaultRcptRaw != nil { + return sourceBlock{}, config.NodeErr(node, "duplicate 'default_destination' block") + } + defaultRcptRaw = node.Children + case "deliver_to", "reroute", "reject": + othersRaw = append(othersRaw, node) + default: + return sourceBlock{}, config.NodeErr(node, "unknown pipeline directive: %s", node.Name) + } + } + + if len(src.perRcpt) == 0 && len(defaultRcptRaw) == 0 { + if len(othersRaw) == 0 { + return sourceBlock{}, fmt.Errorf("empty source block, use 'reject' to reject messages") + } + + var err error + src.defaultRcpt, err = parseMsgPipelineRcptCfg(globals, othersRaw) + return src, err + } else if len(othersRaw) != 0 { + return sourceBlock{}, config.NodeErr(othersRaw[0], "can't put handling directives together with destination rules, did you mean to put it into 'default' block or into all recipient blocks?") + } + + if len(defaultRcptRaw) == 0 { + return sourceBlock{}, config.NodeErr(nodes[0], "missing or empty default destination block, use default_destination { reject } to reject messages") + } + + var err error + src.defaultRcpt, err = parseMsgPipelineRcptCfg(globals, defaultRcptRaw) + return src, err +} + +func parseMsgPipelineRcptCfg(globals map[string]interface{}, nodes []config.Node) (*rcptBlock, error) { + rcpt := rcptBlock{} + for _, node := range nodes { + switch node.Name { + case "check": + checks, err := parseChecksGroup(globals, node) + if err != nil { + return nil, err + } + + rcpt.checks = append(rcpt.checks, checks...) + case "modify": + modifiers, err := parseModifiersGroup(globals, node) + if err != nil { + return nil, err + } + + rcpt.modifiers.Modifiers = append(rcpt.modifiers.Modifiers, modifiers.Modifiers...) + case "deliver_to": + if rcpt.rejectErr != nil { + return nil, config.NodeErr(node, "can't use 'reject' and 'deliver_to' together") + } + + if len(node.Args) == 0 { + return nil, config.NodeErr(node, "required at least one argument") + } + mod, err := modconfig.DeliveryTarget(globals, node.Args, node) + if err != nil { + return nil, err + } + + rcpt.targets = append(rcpt.targets, mod) + case "reroute": + if len(node.Children) == 0 { + return nil, config.NodeErr(node, "missing or empty reroute pipeline configuration") + } + + pipeline, err := New(globals, node.Children) + if err != nil { + return nil, err + } + + rcpt.targets = append(rcpt.targets, pipeline) + case "reject": + if len(rcpt.targets) != 0 { + return nil, config.NodeErr(node, "can't use 'reject' and 'deliver_to' together") + } + + var err error + rcpt.rejectErr, err = parseRejectDirective(node) + if err != nil { + return nil, err + } + default: + return nil, config.NodeErr(node, "invalid directive") + } + } + return &rcpt, nil +} + +func parseRejectDirective(node config.Node) (*exterrors.SMTPError, error) { + code := 554 + enchCode := exterrors.EnhancedCode{5, 7, 0} + msg := "Message rejected due to a local policy" + var err error + switch len(node.Args) { + case 3: + msg = node.Args[2] + if msg == "" { + return nil, config.NodeErr(node, "message can't be empty") + } + fallthrough + case 2: + enchCode, err = parseEnhancedCode(node.Args[1]) + if err != nil { + return nil, config.NodeErr(node, "%v", err) + } + if enchCode[0] != 4 && enchCode[0] != 5 { + return nil, config.NodeErr(node, "enhanced code should use either 4 or 5 as a first number") + } + fallthrough + case 1: + code, err = strconv.Atoi(node.Args[0]) + if err != nil { + return nil, config.NodeErr(node, "invalid error code integer: %v", err) + } + if (code/100) != 4 && (code/100) != 5 { + return nil, config.NodeErr(node, "error code should start with either 4 or 5") + } + case 0: + default: + return nil, config.NodeErr(node, "invalid count of arguments") + } + return &exterrors.SMTPError{ + Code: code, + EnhancedCode: enchCode, + Message: msg, + Reason: "reject directive used", + }, nil +} + +func parseEnhancedCode(s string) (exterrors.EnhancedCode, error) { + parts := strings.Split(s, ".") + if len(parts) != 3 { + return exterrors.EnhancedCode{}, fmt.Errorf("wrong amount of enhanced code parts") + } + + code := exterrors.EnhancedCode{} + for i, part := range parts { + num, err := strconv.Atoi(part) + if err != nil { + return code, err + } + code[i] = num + } + return code, nil +} + +func parseChecksGroup(globals map[string]interface{}, node config.Node) ([]module.Check, error) { + var cg *CheckGroup + err := modconfig.GroupFromNode("checks", node.Args, node, globals, &cg) + if err != nil { + return nil, err + } + return cg.L, nil +} + +func parseModifiersGroup(globals map[string]interface{}, node config.Node) (modify.Group, error) { + // Module object is *modify.Group, not modify.Group. + var mg *modify.Group + err := modconfig.GroupFromNode("modifiers", node.Args, node, globals, &mg) + if err != nil { + return modify.Group{}, err + } + return *mg, nil +} + +func validMatchRule(rule string) bool { + return address.ValidDomain(rule) || address.Valid(rule) +} diff --git a/internal/msgpipeline/config_test.go b/internal/msgpipeline/config_test.go new file mode 100644 index 0000000..24d7e51 --- /dev/null +++ b/internal/msgpipeline/config_test.go @@ -0,0 +1,440 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "reflect" + "strings" + "testing" + + parser "github.com/foxcpp/maddy/framework/cfgparser" + "github.com/foxcpp/maddy/framework/exterrors" +) + +func policyError(code int) error { + return &exterrors.SMTPError{ + Message: "Message rejected due to a local policy", + Code: code, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Reason: "reject directive used", + } +} + +func TestMsgPipelineCfg(t *testing.T) { + cases := []struct { + name string + str string + value msgpipelineCfg + fail bool + }{ + { + name: "basic", + str: ` + source example.com { + destination example.org { + reject 410 + } + default_destination { + reject 420 + } + } + default_source { + destination example.org { + reject 430 + } + default_destination { + reject 440 + } + }`, + value: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "example.com": { + perRcpt: map[string]*rcptBlock{ + "example.org": { + rejectErr: policyError(410), + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: policyError(420), + }, + }, + }, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "example.org": { + rejectErr: policyError(430), + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: policyError(440), + }, + }, + }, + }, + { + name: "implied default destination", + str: ` + source example.com { + reject 410 + } + default_source { + reject 420 + }`, + value: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + rejectErr: policyError(410), + }, + }, + }, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + rejectErr: policyError(420), + }, + }, + }, + }, + { + name: "implied default sender", + str: ` + destination example.com { + reject 410 + } + default_destination { + reject 420 + }`, + value: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "example.com": { + rejectErr: policyError(410), + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: policyError(420), + }, + }, + }, + }, + { + name: "missing default source handler", + str: ` + source example.org { + reject 410 + }`, + fail: true, + }, + { + name: "missing default destination handler", + str: ` + destination example.org { + reject 410 + }`, + fail: true, + }, + { + name: "invalid domain", + str: ` + destination .. { + reject 410 + } + default_destination { + reject 500 + }`, + fail: true, + }, + { + name: "invalid address", + str: ` + destination @example. { + reject 410 + } + default_destination { + reject 500 + }`, + fail: true, + }, + { + name: "invalid address", + str: ` + destination @example. { + reject 421 + } + default_destination { + reject 500 + }`, + fail: true, + }, + { + name: "invalid reject code", + str: ` + destination example.com { + reject 200 + } + default_destination { + reject 500 + }`, + fail: true, + }, + { + name: "destination together with source", + str: ` + destination example.com { + reject 410 + } + source example.org { + reject 420 + } + default_source { + reject 430 + }`, + fail: true, + }, + { + name: "empty destination rule", + str: ` + destination { + reject 410 + } + default_destination { + reject 420 + }`, + fail: true, + }, + } + + for _, case_ := range cases { + t.Run(case_.name, func(t *testing.T) { + cfg, _ := parser.Read(strings.NewReader(case_.str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil && !case_.fail { + t.Fatalf("unexpected parse error: %v", err) + } + if err == nil && case_.fail { + t.Fatalf("unexpected parse success") + } + if case_.fail { + t.Log(err) + return + } + if !reflect.DeepEqual(parsed, case_.value) { + t.Errorf("Wrong parsed configuration") + t.Errorf("Wanted: %+v", case_.value) + t.Errorf("Got: %+v", parsed) + } + }) + } +} + +func TestMsgPipelineCfg_SourceIn(t *testing.T) { + str := ` + source_in dummy { + deliver_to dummy + } + default_source { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.sourceIn) == 0 { + t.Fatalf("missing source_in dummy") + } +} + +func TestMsgPipelineCfg_DestIn(t *testing.T) { + str := ` + destination_in dummy { + deliver_to dummy + } + default_destination { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.defaultSource.rcptIn) == 0 { + t.Fatalf("missing destination_in dummy") + } +} + +func TestMsgPipelineCfg_GlobalChecks(t *testing.T) { + str := ` + check { + test_check + } + default_destination { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.globalChecks) == 0 { + t.Fatalf("missing test_check in globalChecks") + } +} + +func TestMsgPipelineCfg_GlobalChecksMultiple(t *testing.T) { + str := ` + check { + test_check + } + check { + test_check + } + default_destination { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.globalChecks) != 2 { + t.Fatalf("wrong amount of test_check's in globalChecks: %d", len(parsed.globalChecks)) + } +} + +func TestMsgPipelineCfg_SourceChecks(t *testing.T) { + str := ` + source example.org { + check { + test_check + } + + reject 500 + } + default_source { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.perSource["example.org"].checks) == 0 { + t.Fatalf("missing test_check in source checks") + } +} + +func TestMsgPipelineCfg_SourceChecks_Multiple(t *testing.T) { + str := ` + source example.org { + check { + test_check + } + check { + test_check + } + + reject 500 + } + default_source { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.perSource["example.org"].checks) != 2 { + t.Fatalf("wrong amount of test_check's in source checks: %d", len(parsed.perSource["example.org"].checks)) + } +} + +func TestMsgPipelineCfg_RcptChecks(t *testing.T) { + str := ` + destination example.org { + check { + test_check + } + + reject 500 + } + default_destination { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.defaultSource.perRcpt["example.org"].checks) == 0 { + t.Fatalf("missing test_check in rcpt checks") + } +} + +func TestMsgPipelineCfg_RcptChecks_Multiple(t *testing.T) { + str := ` + destination example.org { + check { + test_check + } + check { + test_check + } + + reject 500 + } + default_destination { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.defaultSource.perRcpt["example.org"].checks) != 2 { + t.Fatalf("wrong amount of test_check's in rcpt checks: %d", len(parsed.defaultSource.perRcpt["example.org"].checks)) + } +} diff --git a/internal/msgpipeline/dmarc_test.go b/internal/msgpipeline/dmarc_test.go new file mode 100644 index 0000000..f942baf --- /dev/null +++ b/internal/msgpipeline/dmarc_test.go @@ -0,0 +1,224 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "bufio" + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "net" + "strings" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/emersion/go-smtp" + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func doTestDelivery(t *testing.T, tgt module.DeliveryTarget, from string, to []string, hdr string) (string, error) { + t.Helper() + + IDRaw := sha1.Sum([]byte(t.Name())) + encodedID := hex.EncodeToString(IDRaw[:]) + + body := buffer.MemoryBuffer{Slice: []byte("foobar")} + ctx := module.MsgMetadata{ + DontTraceSender: true, + ID: encodedID, + } + + hdrParsed, err := textproto.ReadHeader(bufio.NewReader(strings.NewReader(hdr))) + if err != nil { + panic(err) + } + + delivery, err := tgt.Start(context.Background(), &ctx, from) + if err != nil { + return encodedID, err + } + for _, rcpt := range to { + if err := delivery.AddRcpt(context.Background(), rcpt, smtp.RcptOptions{}); err != nil { + if err := delivery.Abort(context.Background()); err != nil { + t.Log("delivery.Abort:", err) + } + return encodedID, err + } + } + if err := delivery.Body(context.Background(), hdrParsed, body); err != nil { + if err := delivery.Abort(context.Background()); err != nil { + t.Log("delivery.Abort:", err) + } + return encodedID, err + } + if err := delivery.Commit(context.Background()); err != nil { + return encodedID, err + } + + return encodedID, err +} + +func dmarcResult(t *testing.T, hdr textproto.Header) authres.ResultValue { + field := hdr.Get("Authentication-Results") + if field == "" { + t.Fatalf("No results field") + } + + _, results, err := authres.Parse(field) + if err != nil { + t.Fatalf("Field parse err: %v", err) + } + + for _, res := range results { + dmarcRes, ok := res.(*authres.DMARCResult) + if ok { + return dmarcRes.Value + } + } + + t.Fatalf("No DMARC authres found") + return "" +} + +func TestDMARC(t *testing.T) { + test := func(zones map[string]mockdns.Zone, hdr string, authres []authres.Result, reject, quarantine bool, dmarcRes authres.ResultValue) { + t.Helper() + + tgt := testutils.Target{} + p := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{ + &testutils.Check{ + BodyRes: module.CheckResult{ + AuthResult: authres, + }, + }, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&tgt}, + }, + }, + doDMARC: true, + }, + Log: testutils.Logger(t, "pipeline"), + Resolver: &mockdns.Resolver{Zones: zones}, + } + + _, err := doTestDelivery(t, &p, "test@example.org", []string{"test@example.com"}, hdr) + if reject { + if err == nil { + t.Errorf("expected message to be rejected") + return + } + t.Log(err, exterrors.Fields(err)) + return + } + if err != nil { + t.Errorf("unexpected error: %v %+v", err, exterrors.Fields(err)) + return + } + + if len(tgt.Messages) != 1 { + t.Errorf("got %d messages", len(tgt.Messages)) + return + } + msg := tgt.Messages[0] + + if msg.MsgMeta.Quarantine != quarantine { + t.Errorf("msg.MsgMeta.Quarantine (%v) != quarantine (%v)", msg.MsgMeta.Quarantine, quarantine) + return + } + + res := dmarcResult(t, msg.Header) + if res != dmarcRes { + t.Errorf("expected DMARC result to be '%v', got '%v'", dmarcRes, res) + return + } + } + + // No policy => DMARC 'none' + test(map[string]mockdns.Zone{}, "From: hello@example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, false, false, authres.ResultNone) + + // Policy present & identifiers align => DMARC 'pass' + test(map[string]mockdns.Zone{ + "_dmarc.example.org.": { + TXT: []string{"v=DMARC1; p=none"}, + }, + }, "From: hello@example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, false, false, authres.ResultPass) + + // Policy fetch error => DMARC 'permerror' but the message + // is accepted. + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + Err: errors.New("the dns server is going insane"), + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, false, false, authres.ResultPermError) + + // Policy fetch error => DMARC 'temperror' but the message + // is rejected ("fail closed") + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + Err: &net.DNSError{ + Err: "the dns server is going insane, temporary", + IsTemporary: true, + }, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, true, false, authres.ResultTempError) + + // Misaligned From vs DKIM => DMARC 'fail', policy says to reject + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + TXT: []string{"v=DMARC1; p=reject"}, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, true, false, "") + + // Misaligned From vs DKIM => DMARC 'fail', policy says to quarantine. + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + TXT: []string{"v=DMARC1; p=quarantine"}, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, false, true, authres.ResultFail) +} diff --git a/internal/msgpipeline/metrics.go b/internal/msgpipeline/metrics.go new file mode 100644 index 0000000..691c047 --- /dev/null +++ b/internal/msgpipeline/metrics.go @@ -0,0 +1,47 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import "github.com/prometheus/client_golang/prometheus" + +var ( + checkReject = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "check", + Name: "reject", + Help: "Number of times a check returned 'reject' result (may be more than processed messages if check does so on per-recipient basis)", + }, + []string{"check"}, + ) + checkQuarantined = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "check", + Name: "quarantined", + Help: "Number of times a check returned 'quarantine' result (may be more than processed messages if check does so on per-recipient basis)", + }, + []string{"check"}, + ) +) + +func init() { + prometheus.MustRegister(checkReject) + prometheus.MustRegister(checkQuarantined) +} diff --git a/internal/msgpipeline/modifier_test.go b/internal/msgpipeline/modifier_test.go new file mode 100644 index 0000000..ac3a0d7 --- /dev/null +++ b/internal/msgpipeline/modifier_test.go @@ -0,0 +1,709 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "errors" + "testing" + + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/modify" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestMsgPipeline_SenderModifier(t *testing.T) { + target := testutils.Target{} + modifier := testutils.Modifier{ + InstName: "test_modifier", + MailFrom: map[string]string{ + "sender@example.com": "sender2@example.com", + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{modifier}, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender2@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if modifier.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", modifier.UnclosedStates) + } +} + +func TestMsgPipeline_SenderModifier_Multiple(t *testing.T) { + target := testutils.Target{} + mod1, mod2 := testutils.Modifier{ + InstName: "first_modifier", + MailFrom: map[string]string{ + "sender@example.com": "sender2@example.com", + }, + }, testutils.Modifier{ + InstName: "second_modifier", + MailFrom: map[string]string{ + "sender2@example.com": "sender3@example.com", + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{mod1, mod2}, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender3@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if mod1.UnclosedStates != 0 || mod2.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d, %d", mod1.UnclosedStates, mod2.UnclosedStates) + } +} + +func TestMsgPipeline_SenderModifier_PreDispatch(t *testing.T) { + target := testutils.Target{InstName: "target"} + mod := testutils.Modifier{ + InstName: "test_modifier", + MailFrom: map[string]string{ + "sender@example.com": "sender@example.org", + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{mod}, + }, + perSource: map[string]sourceBlock{ + "example.org": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + defaultSource: sourceBlock{rejectErr: errors.New("default src block used")}, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 1, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if mod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", mod.UnclosedStates) + } +} + +func TestMsgPipeline_SenderModifier_PostDispatch(t *testing.T) { + target := testutils.Target{InstName: "target"} + mod := testutils.Modifier{ + InstName: "test_modifier", + MailFrom: map[string]string{ + "sender@example.org": "sender@example.com", + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "example.org": { + modifiers: modify.Group{ + Modifiers: []module.Modifier{mod}, + }, + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + defaultSource: sourceBlock{rejectErr: errors.New("default src block used")}, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 1, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if mod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", mod.UnclosedStates) + } +} + +func TestMsgPipeline_SenderModifier_PerRcpt(t *testing.T) { + // Modifier below will be no-op due to implementation limitations. + + comTarget, orgTarget := testutils.Target{InstName: "com_target"}, testutils.Target{InstName: "org_target"} + mod := testutils.Modifier{ + InstName: "test_modifier", + MailFrom: map[string]string{ + "sender@example.com": "sender2@example.com", + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "example.com": { + modifiers: modify.Group{ + Modifiers: []module.Modifier{mod}, + }, + targets: []module.DeliveryTarget{&comTarget}, + }, + "example.org": { + modifiers: modify.Group{}, + targets: []module.DeliveryTarget{&orgTarget}, + }, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt@example.com", "rcpt@example.org"}) + + if len(comTarget.Messages) != 1 { + t.Fatalf("wrong amount of messages received for comTarget, want %d, got %d", 1, len(comTarget.Messages)) + } + testutils.CheckTestMessage(t, &comTarget, 0, "sender@example.com", []string{"rcpt@example.com"}) + + if len(orgTarget.Messages) != 1 { + t.Fatalf("wrong amount of messages received for orgTarget, want %d, got %d", 1, len(orgTarget.Messages)) + } + testutils.CheckTestMessage(t, &orgTarget, 0, "sender@example.com", []string{"rcpt@example.org"}) + + if mod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", mod.UnclosedStates) + } +} + +func TestMsgPipeline_RcptModifier(t *testing.T) { + target := testutils.Target{} + mod := testutils.Modifier{ + InstName: "test_modifier", + RcptTo: map[string][]string{ + "rcpt1@example.com": []string{"rcpt1-alias@example.com"}, + "rcpt2@example.com": []string{"rcpt2-alias@example.com"}, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{mod}, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1-alias@example.com", "rcpt2-alias@example.com"}) + + if mod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", mod.UnclosedStates) + } +} + +func TestMsgPipeline_RcptModifier_OriginalRcpt(t *testing.T) { + target := testutils.Target{} + mod := testutils.Modifier{ + InstName: "test_modifier", + RcptTo: map[string][]string{ + "rcpt1@example.com": []string{"rcpt1-alias@example.com"}, + "rcpt2@example.com": []string{"rcpt2-alias@example.com"}, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{mod}, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1-alias@example.com", "rcpt2-alias@example.com"}) + original1 := target.Messages[0].MsgMeta.OriginalRcpts["rcpt1-alias@example.com"] + if original1 != "rcpt1@example.com" { + t.Errorf("wrong OriginalRcpts value for first rcpt, want %s, got %s", "rcpt1@example.com", original1) + } + original2 := target.Messages[0].MsgMeta.OriginalRcpts["rcpt2-alias@example.com"] + if original2 != "rcpt2@example.com" { + t.Errorf("wrong OriginalRcpts value for first rcpt, want %s, got %s", "rcpt2@example.com", original2) + } + + if mod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", mod.UnclosedStates) + } +} + +func TestMsgPipeline_RcptModifier_OriginalRcpt_Multiple(t *testing.T) { + target := testutils.Target{} + mod1, mod2 := testutils.Modifier{ + InstName: "first_modifier", + RcptTo: map[string][]string{ + "rcpt1@example.com": []string{"rcpt1-alias@example.com"}, + "rcpt2@example.com": []string{"rcpt2-alias@example.com"}, + }, + }, testutils.Modifier{ + InstName: "second_modifier", + RcptTo: map[string][]string{ + "rcpt1-alias@example.com": []string{"rcpt1-alias2@example.com"}, + "rcpt2@example.com": []string{"wtf@example.com"}, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{mod1}, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + modifiers: modify.Group{ + Modifiers: []module.Modifier{mod2}, + }, + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1-alias2@example.com", "rcpt2-alias@example.com"}) + original1 := target.Messages[0].MsgMeta.OriginalRcpts["rcpt1-alias2@example.com"] + if original1 != "rcpt1@example.com" { + t.Errorf("wrong OriginalRcpts value for first rcpt, want %s, got %s", "rcpt1@example.com", original1) + } + original2 := target.Messages[0].MsgMeta.OriginalRcpts["rcpt2-alias@example.com"] + if original2 != "rcpt2@example.com" { + t.Errorf("wrong OriginalRcpts value for first rcpt, want %s, got %s", "rcpt2@example.com", original2) + } + + if mod1.UnclosedStates != 0 || mod2.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d, %d", mod1.UnclosedStates, mod2.UnclosedStates) + } +} + +func TestMsgPipeline_RcptModifier_Multiple(t *testing.T) { + target := testutils.Target{} + mod1, mod2 := testutils.Modifier{ + InstName: "first_modifier", + RcptTo: map[string][]string{ + "rcpt1@example.com": []string{"rcpt1-alias@example.com"}, + "rcpt2@example.com": []string{"rcpt2-alias@example.com"}, + }, + }, testutils.Modifier{ + InstName: "second_modifier", + RcptTo: map[string][]string{ + "rcpt1-alias@example.com": []string{"rcpt1-alias2@example.com"}, + "rcpt2@example.com": []string{"wtf@example.com"}, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{mod1, mod2}, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1-alias2@example.com", "rcpt2-alias@example.com"}) + + if mod1.UnclosedStates != 0 || mod2.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d, %d", mod1.UnclosedStates, mod2.UnclosedStates) + } +} + +func TestMsgPipeline_RcptModifier_PreDispatch(t *testing.T) { + target := testutils.Target{} + mod1, mod2 := testutils.Modifier{ + InstName: "first_modifier", + RcptTo: map[string][]string{ + "rcpt1@example.com": []string{"rcpt1-alias@example.com"}, + "rcpt2@example.com": []string{"rcpt2-alias@example.com"}, + }, + }, testutils.Modifier{ + InstName: "second_modifier", + RcptTo: map[string][]string{ + "rcpt1-alias@example.com": []string{"rcpt1-alias2@example.com"}, + "rcpt2@example.com": []string{"wtf@example.com"}, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{mod1}, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + modifiers: modify.Group{Modifiers: []module.Modifier{mod2}}, + perRcpt: map[string]*rcptBlock{ + "rcpt2-alias@example.com": { + targets: []module.DeliveryTarget{&target}, + }, + "rcpt1-alias2@example.com": { + targets: []module.DeliveryTarget{&target}, + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("default rcpt is used"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1-alias2@example.com", "rcpt2-alias@example.com"}) + + if mod1.UnclosedStates != 0 || mod2.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d, %d", mod1.UnclosedStates, mod2.UnclosedStates) + } +} + +func TestMsgPipeline_RcptModifier_PostDispatch(t *testing.T) { + target := testutils.Target{} + mod := testutils.Modifier{ + InstName: "test_modifier", + RcptTo: map[string][]string{ + "rcpt1@example.com": []string{"rcpt1@example.org"}, + "rcpt2@example.com": []string{"rcpt2@example.org"}, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "example.com": { + modifiers: modify.Group{ + Modifiers: []module.Modifier{mod}, + }, + targets: []module.DeliveryTarget{&target}, + }, + "example.org": { + rejectErr: errors.New("wrong rcpt block is used"), + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("default rcpt is used"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1@example.org", "rcpt2@example.org"}) + + if mod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", mod.UnclosedStates) + } +} + +func TestMsgPipeline_GlobalModifier_Errors(t *testing.T) { + target := testutils.Target{} + mod := testutils.Modifier{ + InstName: "test_modifier", + InitErr: errors.New("1"), + MailFromErr: errors.New("2"), + RcptToErr: errors.New("3"), + BodyErr: errors.New("4"), + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{Modifiers: []module.Modifier{&mod}}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + t.Run("init err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.InitErr = nil + + t.Run("mail from err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.MailFromErr = nil + + t.Run("rcpt to err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.RcptToErr = nil + + t.Run("body err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.BodyErr = nil + + t.Run("no err", func(t *testing.T) { + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + }) + + if mod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", mod.UnclosedStates) + } +} + +func TestMsgPipeline_SourceModifier_Errors(t *testing.T) { + target := testutils.Target{} + mod := testutils.Modifier{ + InstName: "test_modifier", + InitErr: errors.New("1"), + MailFromErr: errors.New("2"), + RcptToErr: errors.New("3"), + BodyErr: errors.New("4"), + } + // Added to make sure it is freed properly too. + globalMod := testutils.Modifier{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + globalModifiers: modify.Group{Modifiers: []module.Modifier{&globalMod}}, + defaultSource: sourceBlock{ + modifiers: modify.Group{Modifiers: []module.Modifier{&mod}}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + t.Run("init err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.InitErr = nil + + t.Run("mail from err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.MailFromErr = nil + + t.Run("rcpt to err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.RcptToErr = nil + + t.Run("body err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.BodyErr = nil + + t.Run("no err", func(t *testing.T) { + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + }) + + if mod.UnclosedStates != 0 || globalMod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counters: %d, %d", + mod.UnclosedStates, globalMod.UnclosedStates) + } +} + +func TestMsgPipeline_RcptModifier_Errors(t *testing.T) { + target := testutils.Target{} + mod := testutils.Modifier{ + InstName: "test_modifier", + InitErr: errors.New("1"), + RcptToErr: errors.New("3"), + } + // Added to make sure it is freed properly too. + globalMod := testutils.Modifier{} + sourceMod := testutils.Modifier{} + + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + globalModifiers: modify.Group{Modifiers: []module.Modifier{&globalMod}}, + defaultSource: sourceBlock{ + modifiers: modify.Group{Modifiers: []module.Modifier{&sourceMod}}, + defaultRcpt: &rcptBlock{ + modifiers: modify.Group{Modifiers: []module.Modifier{&mod}}, + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + t.Run("init err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.InitErr = nil + + // MailFromErr test is inapplicable since RewriteSender is not called for per-rcpt + // modifiers. + + t.Run("rcpt to err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.RcptToErr = nil + + // BodyErr test is inapplicable since RewriteBody is not called for per-rcpt + // modifiers. + + t.Run("no err", func(t *testing.T) { + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + }) + + if mod.UnclosedStates != 0 || globalMod.UnclosedStates != 0 || sourceMod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counters: %d, %d, %d", + mod.UnclosedStates, globalMod.UnclosedStates, sourceMod.UnclosedStates) + } +} diff --git a/internal/msgpipeline/module.go b/internal/msgpipeline/module.go new file mode 100644 index 0000000..cf30d22 --- /dev/null +++ b/internal/msgpipeline/module.go @@ -0,0 +1,70 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +type Module struct { + instName string + log log.Logger + *MsgPipeline +} + +func NewModule(modName, instName string, aliases, inlineArgs []string) (module.Module, error) { + return &Module{ + log: log.Logger{Name: "msgpipeline"}, + instName: instName, + }, nil +} + +func (m *Module) Init(cfg *config.Map) error { + var hostname string + cfg.String("hostname", true, true, "", &hostname) + cfg.Bool("debug", true, false, &m.log.Debug) + cfg.AllowUnknown() + other, err := cfg.Process() + if err != nil { + return err + } + + p, err := New(cfg.Globals, other) + if err != nil { + return err + } + m.MsgPipeline = p + m.MsgPipeline.Log = m.log + + return nil +} + +func (m *Module) Name() string { + return "msgpipeline" +} + +func (m *Module) InstanceName() string { + return m.instName +} + +func init() { + module.Register("msgpipeline", NewModule) +} diff --git a/internal/msgpipeline/msgpipeline.go b/internal/msgpipeline/msgpipeline.go new file mode 100644 index 0000000..388caab --- /dev/null +++ b/internal/msgpipeline/msgpipeline.go @@ -0,0 +1,655 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/modify" + "github.com/foxcpp/maddy/internal/target" + "golang.org/x/sync/errgroup" +) + +// MsgPipeline is a object that is responsible for selecting delivery targets +// for the message and running necessary checks and modifiers. +// +// It implements module.DeliveryTarget. +// +// It is not a "module object" and is intended to be used as part of message +// source (Submission, SMTP, JMAP modules) implementation. +type MsgPipeline struct { + msgpipelineCfg + Hostname string + Resolver dns.Resolver + + // Used to indicate the pipeline is handling messages received from the + // external source and not from any other module. That is, this MsgPipeline + // is an instance embedded in endpoint/smtp implementation, for example. + // + // This is a hack since only MsgPipeline can execute some operations at the + // right time but it is not a good idea to execute them multiple multiple + // times for a single message that might be actually handled my multiple + // pipelines via 'msgpipeline' module or 'reroute' directive. + // + // At the moment, the only such operation is the addition of the Received + // header field. See where it happens for explanation on why it is done + // exactly in this place. + FirstPipeline bool + + Log log.Logger +} + +type rcptIn struct { + t module.Table + block *rcptBlock +} + +type sourceBlock struct { + checks []module.Check + modifiers modify.Group + rejectErr error + rcptIn []rcptIn + perRcpt map[string]*rcptBlock + defaultRcpt *rcptBlock +} + +type rcptBlock struct { + checks []module.Check + modifiers modify.Group + rejectErr error + targets []module.DeliveryTarget +} + +func New(globals map[string]interface{}, cfg []config.Node) (*MsgPipeline, error) { + parsedCfg, err := parseMsgPipelineRootCfg(globals, cfg) + return &MsgPipeline{ + msgpipelineCfg: parsedCfg, + Resolver: dns.DefaultResolver(), + }, err +} + +func (d *MsgPipeline) RunEarlyChecks(ctx context.Context, state *module.ConnState) error { + eg, checkCtx := errgroup.WithContext(ctx) + + // TODO: See if there is some point in parallelization of this + // function. + for _, check := range d.globalChecks { + earlyCheck, ok := check.(module.EarlyCheck) + if !ok { + continue + } + + eg.Go(func() error { + return earlyCheck.CheckConnection(checkCtx, state) + }) + } + return eg.Wait() +} + +// Start starts new message delivery, runs connection and sender checks, sender modifiers +// and selects source block from config to use for handling. +// +// Returned module.Delivery implements PartialDelivery. If underlying target doesn't +// support it, msgpipeline will copy the returned error for all recipients handled +// by target. +func (d *MsgPipeline) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + dd := msgpipelineDelivery{ + d: d, + rcptModifiersState: make(map[*rcptBlock]module.ModifierState), + deliveries: make(map[module.DeliveryTarget]*delivery), + msgMeta: msgMeta, + log: target.DeliveryLogger(d.Log, msgMeta), + } + dd.checkRunner = newCheckRunner(msgMeta, dd.log, d.Resolver) + dd.checkRunner.doDMARC = d.doDMARC + + if msgMeta.OriginalRcpts == nil { + msgMeta.OriginalRcpts = map[string]string{} + } + + if err := dd.start(ctx, msgMeta, mailFrom); err != nil { + dd.close() + return nil, err + } + + return &dd, nil +} + +func (dd *msgpipelineDelivery) start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) error { + var err error + + if err := dd.checkRunner.checkConnSender(ctx, dd.d.globalChecks, mailFrom); err != nil { + return err + } + + if mailFrom, err = dd.initRunGlobalModifiers(ctx, msgMeta, mailFrom); err != nil { + return err + } + + sourceBlock, err := dd.srcBlockForAddr(ctx, mailFrom) + if err != nil { + return err + } + if sourceBlock.rejectErr != nil { + dd.log.Debugf("sender %s rejected with error: %v", mailFrom, sourceBlock.rejectErr) + return sourceBlock.rejectErr + } + dd.sourceBlock = sourceBlock + + if err := dd.checkRunner.checkConnSender(ctx, sourceBlock.checks, mailFrom); err != nil { + return err + } + + sourceModifiersState, err := sourceBlock.modifiers.ModStateForMsg(ctx, msgMeta) + if err != nil { + return err + } + mailFrom, err = sourceModifiersState.RewriteSender(ctx, mailFrom) + if err != nil { + return err + } + dd.sourceModifiersState = sourceModifiersState + + dd.sourceAddr = mailFrom + return nil +} + +func (dd *msgpipelineDelivery) initRunGlobalModifiers(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (string, error) { + globalModifiersState, err := dd.d.globalModifiers.ModStateForMsg(ctx, msgMeta) + if err != nil { + return "", err + } + mailFrom, err = globalModifiersState.RewriteSender(ctx, mailFrom) + if err != nil { + globalModifiersState.Close() + return "", err + } + dd.globalModifiersState = globalModifiersState + return mailFrom, nil +} + +func (dd *msgpipelineDelivery) srcBlockForAddr(ctx context.Context, mailFrom string) (sourceBlock, error) { + cleanFrom := mailFrom + if mailFrom != "" { + var err error + cleanFrom, err = address.ForLookup(mailFrom) + if err != nil { + return sourceBlock{}, &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 7}, + Message: "Unable to normalize the sender address", + Err: err, + } + } + } + + for _, srcIn := range dd.d.sourceIn { + _, ok, err := srcIn.t.Lookup(ctx, cleanFrom) + if err != nil { + dd.log.Error("source_in lookup failed", err, "key", cleanFrom) + continue + } + if !ok { + continue + } + return srcIn.block, nil + } + + // First try to match against complete address. + srcBlock, ok := dd.d.perSource[cleanFrom] + if !ok { + // Then try domain-only. + _, domain, err := address.Split(cleanFrom) + // mailFrom != "" is added as a special condition + // instead of extending address.Split because "" + // is not a valid RFC 282 address and only a special + // value for SMTP. + if err != nil && cleanFrom != "" { + return sourceBlock{}, &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 3}, + Message: "Invalid sender address", + Err: err, + Reason: "Can't extract local-part and host-part", + } + } + + // domain is already case-folded and normalized by the message source. + srcBlock, ok = dd.d.perSource[domain] + if !ok { + // Fallback to the default source block. + srcBlock = dd.d.defaultSource + dd.log.Debugf("sender %s matched by default rule", mailFrom) + } else { + dd.log.Debugf("sender %s matched by domain rule '%s'", mailFrom, domain) + } + } else { + dd.log.Debugf("sender %s matched by address rule '%s'", mailFrom, cleanFrom) + } + return srcBlock, nil +} + +type delivery struct { + module.Delivery + // Recipient addresses this delivery object is used for, original values (not modified by RewriteRcpt). + recipients []string +} + +type msgpipelineDelivery struct { + d *MsgPipeline + + globalModifiersState module.ModifierState + sourceModifiersState module.ModifierState + rcptModifiersState map[*rcptBlock]module.ModifierState + + log log.Logger + + sourceAddr string + sourceBlock sourceBlock + + deliveries map[module.DeliveryTarget]*delivery + msgMeta *module.MsgMetadata + checkRunner *checkRunner +} + +func (dd *msgpipelineDelivery) AddRcpt(ctx context.Context, to string, opts smtp.RcptOptions) error { + if err := dd.checkRunner.checkRcpt(ctx, dd.d.globalChecks, to); err != nil { + return err + } + if err := dd.checkRunner.checkRcpt(ctx, dd.sourceBlock.checks, to); err != nil { + return err + } + + originalTo := to + + newTo, err := dd.globalModifiersState.RewriteRcpt(ctx, to) + if err != nil { + return err + } + dd.log.Debugln("global rcpt modifiers:", to, "=>", newTo) + resultTo := newTo + newTo = []string{} + + for _, to = range resultTo { + var tempTo []string + tempTo, err = dd.sourceModifiersState.RewriteRcpt(ctx, to) + if err != nil { + return err + } + newTo = append(newTo, tempTo...) + } + dd.log.Debugln("per-source rcpt modifiers:", to, "=>", newTo) + resultTo = newTo + + for _, to = range resultTo { + wrapErr := func(err error) error { + return exterrors.WithFields(err, map[string]interface{}{ + "effective_rcpt": to, + }) + } + + rcptBlock, err := dd.rcptBlockForAddr(ctx, to) + if err != nil { + return wrapErr(err) + } + + if rcptBlock.rejectErr != nil { + return wrapErr(rcptBlock.rejectErr) + } + + if err := dd.checkRunner.checkRcpt(ctx, rcptBlock.checks, to); err != nil { + return wrapErr(err) + } + + rcptModifiersState, err := dd.getRcptModifiers(ctx, rcptBlock, to) + if err != nil { + return wrapErr(err) + } + + newTo, err = rcptModifiersState.RewriteRcpt(ctx, to) + if err != nil { + rcptModifiersState.Close() + return wrapErr(err) + } + dd.log.Debugln("per-rcpt modifiers:", to, "=>", newTo) + + for _, to = range newTo { + wrapErr = func(err error) error { + return exterrors.WithFields(err, map[string]interface{}{ + "effective_rcpt": to, + }) + } + + if originalTo != to { + dd.msgMeta.OriginalRcpts[to] = originalTo + } + + for _, tgt := range rcptBlock.targets { + // Do not wrap errors coming from nested pipeline target delivery since + // that pipeline itself will insert effective_rcpt field and could do + // its own rewriting - we do not want to hide it from the admin in + // error messages. + wrapErr := wrapErr + if _, ok := tgt.(*MsgPipeline); ok { + wrapErr = func(err error) error { return err } + } + + delivery, err := dd.getDelivery(ctx, tgt) + if err != nil { + return wrapErr(err) + } + + if err := delivery.AddRcpt(ctx, to, opts); err != nil { + return wrapErr(err) + } + delivery.recipients = append(delivery.recipients, originalTo) + } + } + } + + return nil +} + +func (dd *msgpipelineDelivery) Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error { + if err := dd.checkRunner.checkBody(ctx, dd.d.globalChecks, header, body); err != nil { + return err + } + if err := dd.checkRunner.checkBody(ctx, dd.sourceBlock.checks, header, body); err != nil { + return err + } + for blk := range dd.rcptModifiersState { + if err := dd.checkRunner.checkBody(ctx, blk.checks, header, body); err != nil { + return err + } + } + + if dd.d.FirstPipeline { + // Add Received *after* checks to make sure they see the message literally + // how we received it BUT place it below any other field that might be + // added by applyResults (including Authentication-Results) + // per recommendation in RFC 7001, Section 4 (see GH issue #135). + received, err := target.GenerateReceived(ctx, dd.msgMeta, dd.d.Hostname, dd.msgMeta.OriginalFrom) + if err != nil { + return err + } + header.Add("Received", received) + } + + if err := dd.checkRunner.applyResults(dd.d.Hostname, &header); err != nil { + return err + } + + // Run modifiers after Authentication-Results addition to make + // sure signatures, etc will cover it. + if err := dd.globalModifiersState.RewriteBody(ctx, &header, body); err != nil { + return err + } + if err := dd.sourceModifiersState.RewriteBody(ctx, &header, body); err != nil { + return err + } + for _, modifiers := range dd.rcptModifiersState { + if err := modifiers.RewriteBody(ctx, &header, body); err != nil { + return err + } + } + + for _, delivery := range dd.deliveries { + if err := delivery.Body(ctx, header, body); err != nil { + return err + } + dd.log.Debugf("delivery.Body ok, Delivery object = %T", delivery) + } + return nil +} + +// statusCollector wraps StatusCollector and adds reverse translation +// of recipients for all statuses.] +// +// We can't let delivery targets set statuses directly because they see +// modified addresses (RewriteRcpt) and we are supposed to report +// statuses using original values. Additionally, we should still avoid +// collect-and-them-report approach since statuses should be reported +// as soon as possible (that is required by LMTP). +type statusCollector struct { + originalRcpts map[string]string + wrapped module.StatusCollector +} + +func (sc statusCollector) SetStatus(rcptTo string, err error) { + original, ok := sc.originalRcpts[rcptTo] + if ok { + rcptTo = original + } + sc.wrapped.SetStatus(rcptTo, err) +} + +func (dd *msgpipelineDelivery) BodyNonAtomic(ctx context.Context, c module.StatusCollector, header textproto.Header, body buffer.Buffer) { + setStatusAll := func(err error) { + for _, delivery := range dd.deliveries { + for _, rcpt := range delivery.recipients { + c.SetStatus(rcpt, err) + } + } + } + + if err := dd.checkRunner.checkBody(ctx, dd.d.globalChecks, header, body); err != nil { + setStatusAll(err) + return + } + if err := dd.checkRunner.checkBody(ctx, dd.sourceBlock.checks, header, body); err != nil { + setStatusAll(err) + return + } + + // Run modifiers after Authentication-Results addition to make + // sure signatures, etc will cover it. + if err := dd.globalModifiersState.RewriteBody(ctx, &header, body); err != nil { + setStatusAll(err) + return + } + if err := dd.sourceModifiersState.RewriteBody(ctx, &header, body); err != nil { + setStatusAll(err) + return + } + for _, modifiers := range dd.rcptModifiersState { + if err := modifiers.RewriteBody(ctx, &header, body); err != nil { + setStatusAll(err) + return + } + } + + for _, delivery := range dd.deliveries { + partDelivery, ok := delivery.Delivery.(module.PartialDelivery) + if ok { + partDelivery.BodyNonAtomic(ctx, statusCollector{ + originalRcpts: dd.msgMeta.OriginalRcpts, + wrapped: c, + }, header, body) + continue + } + + if err := delivery.Body(ctx, header, body); err != nil { + for _, rcpt := range delivery.recipients { + c.SetStatus(rcpt, err) + } + } + } +} + +func (dd msgpipelineDelivery) Commit(ctx context.Context) error { + dd.close() + + for _, delivery := range dd.deliveries { + if err := delivery.Commit(ctx); err != nil { + // No point in Committing remaining deliveries, everything is broken already. + return err + } + } + return nil +} + +func (dd *msgpipelineDelivery) close() { + dd.checkRunner.close() + + if dd.globalModifiersState != nil { + dd.globalModifiersState.Close() + } + if dd.sourceModifiersState != nil { + dd.sourceModifiersState.Close() + } + for _, modifiers := range dd.rcptModifiersState { + modifiers.Close() + } +} + +func (dd msgpipelineDelivery) Abort(ctx context.Context) error { + dd.close() + + var lastErr error + for _, delivery := range dd.deliveries { + if err := delivery.Abort(ctx); err != nil { + dd.log.Debugf("delivery.Abort failure, Delivery object = %T: %v", delivery, err) + lastErr = err + // Continue anyway and try to Abort all remaining delivery objects. + } + } + return lastErr +} + +func (dd *msgpipelineDelivery) rcptBlockForAddr(ctx context.Context, rcptTo string) (*rcptBlock, error) { + cleanRcpt, err := address.ForLookup(rcptTo) + if err != nil { + return nil, &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 1, 2}, + Message: "Unable to normalize the recipient address", + Err: err, + } + } + + for _, rcptIn := range dd.sourceBlock.rcptIn { + _, ok, err := rcptIn.t.Lookup(ctx, cleanRcpt) + if err != nil { + dd.log.Error("destination_in lookup failed", err, "key", cleanRcpt) + continue + } + if !ok { + continue + } + return rcptIn.block, nil + } + + // First try to match against complete address. + rcptBlock, ok := dd.sourceBlock.perRcpt[cleanRcpt] + if !ok { + // Then try domain-only. + _, domain, err := address.Split(cleanRcpt) + if err != nil { + return nil, &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 3}, + Message: "Invalid recipient address", + Err: err, + Reason: "Can't extract local-part and host-part", + } + } + + // domain is already case-folded and normalized because it is a part of + // cleanRcpt. + rcptBlock, ok = dd.sourceBlock.perRcpt[domain] + if !ok { + // Fallback to the default source block. + rcptBlock = dd.sourceBlock.defaultRcpt + dd.log.Debugf("recipient %s matched by default rule (clean = %s)", rcptTo, cleanRcpt) + } else { + dd.log.Debugf("recipient %s matched by domain rule '%s'", rcptTo, domain) + } + } else { + dd.log.Debugf("recipient %s matched by address rule '%s'", rcptTo, cleanRcpt) + } + return rcptBlock, nil +} + +func (dd *msgpipelineDelivery) getRcptModifiers(ctx context.Context, rcptBlock *rcptBlock, rcptTo string) (module.ModifierState, error) { + rcptModifiersState, ok := dd.rcptModifiersState[rcptBlock] + if ok { + return rcptModifiersState, nil + } + + rcptModifiersState, err := rcptBlock.modifiers.ModStateForMsg(ctx, dd.msgMeta) + if err != nil { + return nil, err + } + + newSender, err := rcptModifiersState.RewriteSender(ctx, dd.sourceAddr) + if err == nil && newSender != dd.sourceAddr { + dd.log.Msg("Per-recipient modifier changed sender address. This is not supported and will "+ + "be ignored.", "rcpt", rcptTo, "originalFrom", dd.sourceAddr, "modifiedFrom", newSender) + } + + dd.rcptModifiersState[rcptBlock] = rcptModifiersState + return rcptModifiersState, nil +} + +func (dd *msgpipelineDelivery) getDelivery(ctx context.Context, tgt module.DeliveryTarget) (*delivery, error) { + delivery_, ok := dd.deliveries[tgt] + if ok { + return delivery_, nil + } + + deliveryObj, err := tgt.Start(ctx, dd.msgMeta, dd.sourceAddr) + if err != nil { + dd.log.Debugf("tgt.Start(%s) failure, target = %s: %v", dd.sourceAddr, objectName(tgt), err) + return nil, err + } + delivery_ = &delivery{Delivery: deliveryObj} + + dd.log.Debugf("tgt.Start(%s) ok, target = %s", dd.sourceAddr, objectName(tgt)) + + dd.deliveries[tgt] = delivery_ + return delivery_, nil +} + +// Mock returns a MsgPipeline that merely delivers messages to a specified target +// and runs a set of checks. +// +// It is meant for use in tests for modules that embed a pipeline object. +func Mock(tgt module.DeliveryTarget, globalChecks []module.Check) *MsgPipeline { + return &MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: globalChecks, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{tgt}, + }, + }, + }, + } +} diff --git a/internal/msgpipeline/msgpipeline_test.go b/internal/msgpipeline/msgpipeline_test.go new file mode 100644 index 0000000..9899eb0 --- /dev/null +++ b/internal/msgpipeline/msgpipeline_test.go @@ -0,0 +1,706 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "context" + "errors" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/modify" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestMsgPipeline_AllToTarget(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) +} + +func TestMsgPipeline_PerSourceDomainSplit(t *testing.T) { + orgTarget, comTarget := testutils.Target{InstName: "orgTarget"}, testutils.Target{InstName: "comTarget"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&comTarget}, + }, + }, + "example.org": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&orgTarget}, + }, + }, + }, + defaultSource: sourceBlock{rejectErr: errors.New("default src block used")}, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + testutils.DoTestDelivery(t, &d, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(comTarget.Messages) != 1 { + t.Fatalf("wrong amount of messages received for comTarget, want %d, got %d", 1, len(comTarget.Messages)) + } + testutils.CheckTestMessage(t, &comTarget, 0, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(orgTarget.Messages) != 1 { + t.Fatalf("wrong amount of messages received for orgTarget, want %d, got %d", 1, len(orgTarget.Messages)) + } + testutils.CheckTestMessage(t, &orgTarget, 0, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}) +} + +func TestMsgPipeline_SourceIn(t *testing.T) { + tblTarget, comTarget := testutils.Target{InstName: "tblTarget"}, testutils.Target{InstName: "comTarget"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + sourceIn: []sourceIn{ + { + t: testutils.Table{}, + block: sourceBlock{rejectErr: errors.New("non-matching block was used")}, + }, + { + t: testutils.Table{Err: errors.New("this one will fail")}, + block: sourceBlock{rejectErr: errors.New("failing block was used")}, + }, + { + t: testutils.Table{ + M: map[string]string{ + "specific@example.com": "", + }, + }, + block: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&tblTarget}, + }, + }, + }, + }, + perSource: map[string]sourceBlock{ + "example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&comTarget}, + }, + }, + }, + defaultSource: sourceBlock{rejectErr: errors.New("default src block used")}, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt@example.com"}) + testutils.DoTestDelivery(t, &d, "specific@example.com", []string{"rcpt@example.com"}) + + if len(comTarget.Messages) != 1 { + t.Fatalf("wrong amount of messages received for comTarget, want %d, got %d", 1, len(comTarget.Messages)) + } + testutils.CheckTestMessage(t, &comTarget, 0, "sender@example.com", []string{"rcpt@example.com"}) + + if len(tblTarget.Messages) != 1 { + t.Fatalf("wrong amount of messages received for orgTarget, want %d, got %d", 1, len(tblTarget.Messages)) + } + testutils.CheckTestMessage(t, &tblTarget, 0, "specific@example.com", []string{"rcpt@example.com"}) +} + +func TestMsgPipeline_EmptyMAILFROM(t *testing.T) { + target := testutils.Target{InstName: "target"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 1, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "", []string{"rcpt1@example.com", "rcpt2@example.com"}) +} + +func TestMsgPipeline_EmptyMAILFROM_ExplicitDest(t *testing.T) { + target := testutils.Target{InstName: "target"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + defaultSource: sourceBlock{rejectErr: errors.New("default src block used")}, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 1, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "", []string{"rcpt1@example.com", "rcpt2@example.com"}) +} + +func TestMsgPipeline_PerRcptAddrSplit(t *testing.T) { + target1, target2 := testutils.Target{InstName: "target1"}, testutils.Target{InstName: "target2"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "rcpt1@example.com": { + targets: []module.DeliveryTarget{&target1}, + }, + "rcpt2@example.com": { + targets: []module.DeliveryTarget{&target2}, + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("defaultRcpt block used"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com"}) + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt2@example.com"}) + + if len(target1.Messages) != 1 { + t.Errorf("wrong amount of messages received for target1, want %d, got %d", 1, len(target1.Messages)) + } + testutils.CheckTestMessage(t, &target1, 0, "sender@example.com", []string{"rcpt1@example.com"}) + + if len(target2.Messages) != 1 { + t.Errorf("wrong amount of messages received for target1, want %d, got %d", 1, len(target2.Messages)) + } + testutils.CheckTestMessage(t, &target2, 0, "sender@example.com", []string{"rcpt2@example.com"}) +} + +func TestMsgPipeline_PerRcptDomainSplit(t *testing.T) { + target1, target2 := testutils.Target{InstName: "target1"}, testutils.Target{InstName: "target2"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "example.com": { + targets: []module.DeliveryTarget{&target1}, + }, + "example.org": { + targets: []module.DeliveryTarget{&target2}, + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("defaultRcpt block used"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.org"}) + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.org", "rcpt2@example.com"}) + + if len(target1.Messages) != 2 { + t.Errorf("wrong amount of messages received for target1, want %d, got %d", 2, len(target1.Messages)) + } + testutils.CheckTestMessage(t, &target1, 0, "sender@example.com", []string{"rcpt1@example.com"}) + testutils.CheckTestMessage(t, &target1, 1, "sender@example.com", []string{"rcpt2@example.com"}) + + if len(target2.Messages) != 2 { + t.Errorf("wrong amount of messages received for target2, want %d, got %d", 2, len(target2.Messages)) + } + testutils.CheckTestMessage(t, &target2, 0, "sender@example.com", []string{"rcpt2@example.org"}) + testutils.CheckTestMessage(t, &target2, 1, "sender@example.com", []string{"rcpt1@example.org"}) +} + +func TestMsgPipeline_DestInSplit(t *testing.T) { + target1, target2 := testutils.Target{InstName: "target1"}, testutils.Target{InstName: "target2"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + rcptIn: []rcptIn{ + { + t: testutils.Table{}, + block: &rcptBlock{rejectErr: errors.New("non-matching block was used")}, + }, + { + t: testutils.Table{Err: errors.New("nope")}, + block: &rcptBlock{rejectErr: errors.New("failing block was used")}, + }, + { + t: testutils.Table{ + M: map[string]string{ + "specific@example.com": "", + }, + }, + block: &rcptBlock{ + targets: []module.DeliveryTarget{&target2}, + }, + }, + }, + perRcpt: map[string]*rcptBlock{ + "example.com": { + targets: []module.DeliveryTarget{&target1}, + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("defaultRcpt block used"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "specific@example.com"}) + + if len(target1.Messages) != 1 { + t.Errorf("wrong amount of messages received for target1, want %d, got %d", 1, len(target1.Messages)) + } + testutils.CheckTestMessage(t, &target1, 0, "sender@example.com", []string{"rcpt1@example.com"}) + + if len(target2.Messages) != 1 { + t.Errorf("wrong amount of messages received for target2, want %d, got %d", 1, len(target2.Messages)) + } + testutils.CheckTestMessage(t, &target2, 0, "sender@example.com", []string{"specific@example.com"}) +} + +func TestMsgPipeline_PerSourceAddrAndDomainSplit(t *testing.T) { + target1, target2 := testutils.Target{InstName: "target1"}, testutils.Target{InstName: "target2"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "sender1@example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target1}, + }, + }, + "example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target2}, + }, + }, + }, + defaultSource: sourceBlock{rejectErr: errors.New("default src block used")}, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender1@example.com", []string{"rcpt@example.com"}) + testutils.DoTestDelivery(t, &d, "sender2@example.com", []string{"rcpt@example.com"}) + + if len(target1.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target1, want %d, got %d", 1, len(target1.Messages)) + } + testutils.CheckTestMessage(t, &target1, 0, "sender1@example.com", []string{"rcpt@example.com"}) + + if len(target2.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target2, want %d, got %d", 1, len(target2.Messages)) + } + testutils.CheckTestMessage(t, &target2, 0, "sender2@example.com", []string{"rcpt@example.com"}) +} + +func TestMsgPipeline_PerSourceReject(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "sender1@example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + "example.com": { + perRcpt: map[string]*rcptBlock{}, + rejectErr: errors.New("go away"), + }, + }, + defaultSource: sourceBlock{rejectErr: errors.New("go away")}, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender1@example.com", []string{"rcpt@example.com"}) + + _, err := d.Start(context.Background(), &module.MsgMetadata{ID: "testing"}, "sender2@example.com") + if err == nil { + t.Error("expected error for delivery.Start, got nil") + } + + _, err = d.Start(context.Background(), &module.MsgMetadata{ID: "testing"}, "sender2@example.org") + if err == nil { + t.Error("expected error for delivery.Start, got nil") + } +} + +func TestMsgPipeline_PerRcptReject(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "rcpt1@example.com": { + targets: []module.DeliveryTarget{&target}, + }, + "example.com": { + rejectErr: errors.New("go away"), + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("go away"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + delivery, err := d.Start(context.Background(), &module.MsgMetadata{ID: "testing"}, "sender@example.com") + if err != nil { + t.Fatalf("unexpected Start err: %v", err) + } + defer func() { + if err := delivery.Abort(context.Background()); err != nil { + t.Fatalf("unexpected Abort err: %v", err) + } + }() + + if err := delivery.AddRcpt(context.Background(), "rcpt2@example.com", smtp.RcptOptions{}); err == nil { + t.Fatalf("expected error for delivery.AddRcpt(rcpt2@example.com), got nil") + } + if err := delivery.AddRcpt(context.Background(), "rcpt1@example.com", smtp.RcptOptions{}); err != nil { + t.Fatalf("unexpected AddRcpt err for %s: %v", "rcpt1@example.com", err) + } + if err := delivery.Body(context.Background(), textproto.Header{}, buffer.MemoryBuffer{Slice: []byte("foobar")}); err != nil { + t.Fatalf("unexpected Body err: %v", err) + } + if err := delivery.Commit(context.Background()); err != nil { + t.Fatalf("unexpected Commit err: %v", err) + } +} + +func TestMsgPipeline_PostmasterRcpt(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "postmaster": { + targets: []module.DeliveryTarget{&target}, + }, + "example.com": { + rejectErr: errors.New("go away"), + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("go away"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "disappointed-user@example.com", []string{"postmaster"}) + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 1, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "disappointed-user@example.com", []string{"postmaster"}) +} + +func TestMsgPipeline_PostmasterSrc(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "postmaster": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + "example.com": { + rejectErr: errors.New("go away"), + }, + }, + defaultSource: sourceBlock{ + rejectErr: errors.New("go away"), + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "postmaster", []string{"disappointed-user@example.com"}) + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 1, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "postmaster", []string{"disappointed-user@example.com"}) +} + +func TestMsgPipeline_CaseInsensetiveMatch_Src(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "postmaster": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + "sender@example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + "example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + defaultSource: sourceBlock{ + rejectErr: errors.New("go away"), + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "POSTMastER", []string{"disappointed-user@example.com"}) + testutils.DoTestDelivery(t, &d, "SenDeR@EXAMPLE.com", []string{"disappointed-user@example.com"}) + testutils.DoTestDelivery(t, &d, "sender@exAMPle.com", []string{"disappointed-user@example.com"}) + if len(target.Messages) != 3 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 3, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "POSTMastER", []string{"disappointed-user@example.com"}) + testutils.CheckTestMessage(t, &target, 1, "SenDeR@EXAMPLE.com", []string{"disappointed-user@example.com"}) + testutils.CheckTestMessage(t, &target, 2, "sender@exAMPle.com", []string{"disappointed-user@example.com"}) +} + +func TestMsgPipeline_CaseInsensetiveMatch_Rcpt(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "postmaster": { + targets: []module.DeliveryTarget{&target}, + }, + "sender@example.com": { + targets: []module.DeliveryTarget{&target}, + }, + "example.com": { + targets: []module.DeliveryTarget{&target}, + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("wtf"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"POSTMastER"}) + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"SenDeR@EXAMPLE.com"}) + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"sender@exAMPle.com"}) + if len(target.Messages) != 3 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 3, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"POSTMastER"}) + testutils.CheckTestMessage(t, &target, 1, "sender@example.com", []string{"SenDeR@EXAMPLE.com"}) + testutils.CheckTestMessage(t, &target, 2, "sender@example.com", []string{"sender@exAMPle.com"}) +} + +func TestMsgPipeline_UnicodeNFC_Rcpt(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "rcpt@é.example.com": { + targets: []module.DeliveryTarget{&target}, + }, + "é.example.com": { + targets: []module.DeliveryTarget{&target}, + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("wtf"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt@E\u0301.EXAMPLE.com"}) + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"f@E\u0301.exAMPle.com"}) + if len(target.Messages) != 2 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 2, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt@E\u0301.EXAMPLE.com"}) + testutils.CheckTestMessage(t, &target, 1, "sender@example.com", []string{"f@E\u0301.exAMPle.com"}) +} + +func TestMsgPipeline_MalformedSource(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "postmaster": { + targets: []module.DeliveryTarget{&target}, + }, + "sender@example.com": { + targets: []module.DeliveryTarget{&target}, + }, + "example.com": { + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + // Simple checks for violations that can make msgpipeline misbehave. + for _, addr := range []string{"not_postmaster_but_no_at_sign", "@no_mailbox", "no_domain@"} { + _, err := d.Start(context.Background(), &module.MsgMetadata{ID: "testing"}, addr) + if err == nil { + t.Errorf("%s is accepted as valid address", addr) + } + } +} + +func TestMsgPipeline_TwoRcptToOneTarget(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "example.com": { + targets: []module.DeliveryTarget{&target}, + }, + "example.org": { + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"recipient@example.com", "recipient@example.org"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 1, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"recipient@example.com", "recipient@example.org"}) +} + +func TestMsgPipeline_multi_alias(t *testing.T) { + target1, target2 := testutils.Target{InstName: "target1"}, testutils.Target{InstName: "target2"} + mod := testutils.Modifier{ + RcptTo: map[string][]string{ + "recipient@example.com": []string{ + "recipient-1@example.org", + "recipient-2@example.net", + }, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + modifiers: modify.Group{ + Modifiers: []module.Modifier{mod}, + }, + perRcpt: map[string]*rcptBlock{ + "example.org": { + targets: []module.DeliveryTarget{&target1}, + }, + "example.net": { + targets: []module.DeliveryTarget{&target2}, + }, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"recipient@example.com"}) + + if len(target1.Messages) != 1 { + t.Errorf("wrong amount of messages received for target1, want %d, got %d", 1, len(target1.Messages)) + } + testutils.CheckTestMessage(t, &target1, 0, "sender@example.com", []string{"recipient-1@example.org"}) + + if len(target2.Messages) != 1 { + t.Errorf("wrong amount of messages received for target1, want %d, got %d", 1, len(target2.Messages)) + } + testutils.CheckTestMessage(t, &target2, 0, "sender@example.com", []string{"recipient-2@example.net"}) +} diff --git a/internal/msgpipeline/objname.go b/internal/msgpipeline/objname.go new file mode 100644 index 0000000..9fc1b27 --- /dev/null +++ b/internal/msgpipeline/objname.go @@ -0,0 +1,46 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "fmt" + + "github.com/foxcpp/maddy/framework/module" +) + +// objectName returns a new that is usable to identify the used external +// component (module or some stub) in debug logs. +func objectName(x interface{}) string { + mod, ok := x.(module.Module) + if ok { + return mod.Name() + ":" + mod.InstanceName() + } + + _, pipeline := x.(*MsgPipeline) + if pipeline { + return "reroute" + } + + str, ok := x.(fmt.Stringer) + if ok { + return str.String() + } + + return fmt.Sprintf("%T", x) +} diff --git a/internal/msgpipeline/regress_test.go b/internal/msgpipeline/regress_test.go new file mode 100644 index 0000000..80e92ef --- /dev/null +++ b/internal/msgpipeline/regress_test.go @@ -0,0 +1,137 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "testing" + + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestMsgPipeline_Issue161(t *testing.T) { + target := testutils.Target{} + check1, check2 := testutils.Check{}, testutils.Check{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&check1}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + checks: []module.Check{&check2}, + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "whatever@whatever", []string{"whatever@whatever"}) + + if check2.ConnCalls != 1 { + t.Errorf("CheckConnection called %d times", check2.ConnCalls) + } + if check2.SenderCalls != 1 { + t.Errorf("CheckSender called %d times", check2.SenderCalls) + } + if check2.RcptCalls != 1 { + t.Errorf("CheckRcpt called %d times", check2.RcptCalls) + } + if check2.BodyCalls != 1 { + t.Errorf("CheckBody called %d times", check2.BodyCalls) + } + + if check1.UnclosedStates != 0 || check2.UnclosedStates != 0 { + t.Fatalf("checks state objects leak or double-closed, alive counters: %v, %v", check1.UnclosedStates, check2.UnclosedStates) + } +} + +func TestMsgPipeline_Issue161_2(t *testing.T) { + target := testutils.Target{} + check1, check2 := testutils.Check{}, testutils.Check{InstName: "check2"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&check1}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + checks: []module.Check{&check1}, + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + checks: []module.Check{&check2}, + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "whatever@whatever", []string{"whatever@whatever"}) + + if check2.ConnCalls != 1 { + t.Errorf("CheckConnection called %d times", check2.ConnCalls) + } + if check2.SenderCalls != 1 { + t.Errorf("CheckSender called %d times", check2.SenderCalls) + } + if check2.RcptCalls != 1 { + t.Errorf("CheckRcpt called %d times", check2.RcptCalls) + } + + if check1.UnclosedStates != 0 || check2.UnclosedStates != 0 { + t.Fatalf("checks state objects leak or double-closed, alive counters: %v, %v", check1.UnclosedStates, check2.UnclosedStates) + } +} + +func TestMsgPipeline_Issue161_3(t *testing.T) { + target := testutils.Target{} + check1, check2 := testutils.Check{}, testutils.Check{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&check1, &check2}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "whatever@whatever", []string{"whatever@whatever"}) + + if check2.ConnCalls != 1 { + t.Errorf("CheckConnection called %d times", check2.ConnCalls) + } + if check2.SenderCalls != 1 { + t.Errorf("CheckSender called %d times", check2.SenderCalls) + } + if check2.RcptCalls != 1 { + t.Errorf("CheckRcpt called %d times", check2.RcptCalls) + } + if check2.BodyCalls != 1 { + t.Errorf("CheckBody called %d times", check2.BodyCalls) + } + + if check1.UnclosedStates != 0 || check2.UnclosedStates != 0 { + t.Fatalf("checks state objects leak or double-closed, alive counters: %v, %v", check1.UnclosedStates, check2.UnclosedStates) + } +} diff --git a/internal/proxy_protocol/proxy_protocol.go b/internal/proxy_protocol/proxy_protocol.go new file mode 100644 index 0000000..1a3a787 --- /dev/null +++ b/internal/proxy_protocol/proxy_protocol.go @@ -0,0 +1,86 @@ +package proxy_protocol + +import ( + "crypto/tls" + "net" + "strings" + + "github.com/c0va23/go-proxyprotocol" + "github.com/foxcpp/maddy/framework/config" + tls2 "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/log" +) + +type ProxyProtocol struct { + trust []net.IPNet + tlsConfig *tls.Config +} + +func ProxyProtocolDirective(_ *config.Map, node config.Node) (interface{}, error) { + p := ProxyProtocol{} + + childM := config.NewMap(nil, node) + var trustList []string + + childM.StringList("trust", false, false, nil, &trustList) + childM.Custom("tls", true, false, nil, tls2.TLSDirective, &p.tlsConfig) + + if _, err := childM.Process(); err != nil { + return nil, err + } + + if len(node.Args) > 0 { + if trustList == nil { + trustList = make([]string, 0) + } + trustList = append(trustList, node.Args...) + } + + for _, trust := range trustList { + if !strings.Contains(trust, "/") { + trust += "/32" + } + _, ipNet, err := net.ParseCIDR(trust) + if err != nil { + return nil, err + } + p.trust = append(p.trust, *ipNet) + } + + return &p, nil +} + +func NewListener(inner net.Listener, p *ProxyProtocol, logger log.Logger) net.Listener { + var listener net.Listener + + sourceChecker := func(upstream net.Addr) (bool, error) { + if tcpAddr, ok := upstream.(*net.TCPAddr); ok { + if len(p.trust) == 0 { + return true, nil + } + for _, trusted := range p.trust { + if trusted.Contains(tcpAddr.IP) { + return true, nil + } + } + } else if _, ok := upstream.(*net.UnixAddr); ok { + // UNIX local socket connection, always trusted + return true, nil + } + + logger.Printf("proxy_protocol: connection from untrusted source %s", upstream) + return false, nil + } + + listener = proxyprotocol.NewDefaultListener(inner). + WithLogger(proxyprotocol.LoggerFunc(func(format string, v ...interface{}) { + logger.Debugf("proxy_protocol: "+format, v...) + })). + WithSourceChecker(sourceChecker) + + if p.tlsConfig != nil { + listener = tls.NewListener(listener, p.tlsConfig) + } + + return listener +} diff --git a/internal/smtpconn/pool/pool.go b/internal/smtpconn/pool/pool.go new file mode 100644 index 0000000..4b700ee --- /dev/null +++ b/internal/smtpconn/pool/pool.go @@ -0,0 +1,211 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package pool + +import ( + "context" + "sync" + "time" +) + +type Conn interface { + Usable() bool + LastUseAt() time.Time + Close() error +} + +type Config struct { + New func(ctx context.Context, key string) (Conn, error) + MaxKeys int + MaxConnsPerKey int + MaxConnLifetimeSec int64 + StaleKeyLifetimeSec int64 +} + +type slot struct { + c chan Conn + // To keep slot size smaller it is just a unix timestamp. + lastUse int64 +} + +type P struct { + cfg Config + keys map[string]slot + keysLock sync.Mutex + + cleanupStop chan struct{} +} + +func New(cfg Config) *P { + if cfg.New == nil { + cfg.New = func(context.Context, string) (Conn, error) { + return nil, nil + } + } + + p := &P{ + cfg: cfg, + keys: make(map[string]slot, cfg.MaxKeys), + cleanupStop: make(chan struct{}), + } + + go p.cleanUpTick(p.cleanupStop) + + return p +} + +func (p *P) cleanUpTick(stop chan struct{}) { + ctx := context.Background() + tick := time.NewTicker(time.Minute) + defer tick.Stop() + + for { + select { + case <-tick.C: + p.CleanUp(ctx) + case <-stop: + return + } + } +} + +func (p *P) CleanUp(ctx context.Context) { + p.keysLock.Lock() + defer p.keysLock.Unlock() + + for k, v := range p.keys { + if v.lastUse+p.cfg.StaleKeyLifetimeSec > time.Now().Unix() { + continue + } + + close(v.c) + for conn := range v.c { + go conn.Close() + } + delete(p.keys, k) + } +} + +func (p *P) Get(ctx context.Context, key string) (Conn, error) { + p.keysLock.Lock() + + bucket, ok := p.keys[key] + if !ok { + p.keysLock.Unlock() + return p.cfg.New(ctx, key) + } + + if time.Now().Unix()-bucket.lastUse > p.cfg.MaxConnLifetimeSec { + // Drop bucket. + delete(p.keys, key) + close(bucket.c) + + // Close might take some time, unlock early. + p.keysLock.Unlock() + + for conn := range bucket.c { + conn.Close() + } + + return p.cfg.New(ctx, key) + } + + p.keysLock.Unlock() + + for { + var conn Conn + select { + case conn, ok = <-bucket.c: + if !ok { + return p.cfg.New(ctx, key) + } + default: + return p.cfg.New(ctx, key) + } + + if !conn.Usable() { + // Close might take some time, run in parallel. + go conn.Close() + continue + } + if conn.LastUseAt().Add(time.Duration(p.cfg.MaxConnLifetimeSec) * time.Second).Before(time.Now()) { + go conn.Close() + continue + } + + return conn, nil + } +} + +func (p *P) Return(key string, c Conn) { + p.keysLock.Lock() + defer p.keysLock.Unlock() + + if p.keys == nil { + return + } + + bucket, ok := p.keys[key] + if !ok { + // Garbage-collect stale buckets. + if len(p.keys) == p.cfg.MaxKeys { + for k, v := range p.keys { + if v.lastUse+p.cfg.StaleKeyLifetimeSec > time.Now().Unix() { + continue + } + delete(p.keys, k) + close(v.c) + + for conn := range v.c { + conn.Close() + } + } + } + + bucket = slot{ + c: make(chan Conn, p.cfg.MaxConnsPerKey), + lastUse: time.Now().Unix(), + } + p.keys[key] = bucket + } + + select { + case bucket.c <- c: + bucket.lastUse = time.Now().Unix() + default: + // Let it go, let it go... + go c.Close() + } +} + +func (p *P) Close() { + p.cleanupStop <- struct{}{} + + p.keysLock.Lock() + defer p.keysLock.Unlock() + + for k, v := range p.keys { + close(v.c) + for conn := range v.c { + conn.Close() + } + delete(p.keys, k) + } + p.keys = nil +} diff --git a/internal/smtpconn/smtpconn.go b/internal/smtpconn/smtpconn.go new file mode 100644 index 0000000..ec42974 --- /dev/null +++ b/internal/smtpconn/smtpconn.go @@ -0,0 +1,558 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package smtpconn contains the code shared between target.smtp and +// remote modules. +// +// It implements the wrapper over the SMTP connection (go-smtp.Client) object +// with the following features added: +// - Logging of certain errors (e.g. QUIT command errors) +// - Wrapping of returned errors using the exterrors package. +// - SMTPUTF8/IDNA support. +// - TLS support mode (don't use, attempt, require). +package smtpconn + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "runtime/trace" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" +) + +// The C object represents the SMTP connection and is a wrapper around +// go-smtp.Client with additional maddy-specific logic. +// +// Currently, the C object represents one session and cannot be reused. +type C struct { + // Dialer to use to estabilish new network connections. Set to net.Dialer + // DialContext by New. + Dialer func(ctx context.Context, network, addr string) (net.Conn, error) + + // Timeout for most session commands (EHLO, MAIL, RCPT, DATA, STARTTLS). + // Set to 5 mins by New. + CommandTimeout time.Duration + + // Timeout for the initial TCP connection establishment. + ConnectTimeout time.Duration + + // Timeout for the final dot. Set to 12 mins by New. + // (see go-smtp source for explanation of used defaults). + SubmissionTimeout time.Duration + + // Hostname to sent in the EHLO/HELO command. Set to + // 'localhost.localdomain' by New. Expected to be encoded in ACE form. + Hostname string + + // tls.Config to use. Can be nil if no special changes are required. + TLSConfig *tls.Config + + // Logger to use for debug log and certain errors. + Log log.Logger + + // Include the remote server address in SMTP status messages in the form + // "ADDRESS said: ..." + AddrInSMTPMsg bool + + conn net.Conn + serverName string + cl *smtp.Client + rcpts []string + lmtp bool +} + +// New creates the new instance of the C object, populating the required fields +// with resonable default values. +func New() *C { + return &C{ + Dialer: (&net.Dialer{}).DialContext, + ConnectTimeout: 5 * time.Minute, + CommandTimeout: 5 * time.Minute, + SubmissionTimeout: 12 * time.Minute, + TLSConfig: &tls.Config{}, + Hostname: "localhost.localdomain", + } +} + +func (c *C) wrapClientErr(err error, serverName string) error { + if err == nil { + return nil + } + + switch err := err.(type) { + case TLSError: + return err + case *exterrors.SMTPError: + return err + case *smtp.SMTPError: + msg := err.Message + if c.AddrInSMTPMsg { + msg = serverName + " said: " + err.Message + } + + if err.Code == 552 { + err.Code = 452 + err.EnhancedCode[0] = 4 + c.Log.Msg("SMTP code 552 rewritten to 452 per RFC 5321 Section 4.5.3.1.10") + } + + return &exterrors.SMTPError{ + Code: err.Code, + EnhancedCode: exterrors.EnhancedCode(err.EnhancedCode), + Message: msg, + Misc: map[string]interface{}{ + "remote_server": serverName, + }, + Err: err, + } + case *net.OpError: + if _, ok := err.Err.(*net.DNSError); ok { + reason, misc := exterrors.UnwrapDNSErr(err) + misc["remote_server"] = err.Addr + misc["io_op"] = err.Op + return &exterrors.SMTPError{ + Code: exterrors.SMTPCode(err, 450, 550), + EnhancedCode: exterrors.SMTPEnchCode(err, exterrors.EnhancedCode{0, 4, 4}), + Message: "DNS error", + Err: err, + Reason: reason, + Misc: misc, + } + } + return &exterrors.SMTPError{ + Code: 450, + EnhancedCode: exterrors.EnhancedCode{4, 4, 2}, + Message: "Network I/O error", + Err: err, + Misc: map[string]interface{}{ + "remote_addr": err.Addr, + "io_op": err.Op, + }, + } + default: + return exterrors.WithFields(err, map[string]interface{}{ + "remote_server": serverName, + }) + } +} + +// Connect actually estabilishes the network connection with the remote host, +// executes HELO/EHLO and optionally STARTTLS command. +func (c *C) Connect(ctx context.Context, endp config.Endpoint, starttls bool, tlsConfig *tls.Config) (didTLS bool, err error) { + didTLS, cl, conn, err := c.attemptConnect(ctx, false, endp, starttls, tlsConfig) + if err != nil { + return false, c.wrapClientErr(err, endp.Host) + } + + c.serverName = endp.Host + c.cl = cl + c.conn = conn + + c.Log.DebugMsg("connected", "remote_server", c.serverName, + "local_addr", c.LocalAddr(), "remote_addr", c.RemoteAddr()) + + return didTLS, nil +} + +// ConnectLMTP estabilishes the network connection with the remote host and +// sends LHLO command, negotiating LMTP use. +func (c *C) ConnectLMTP(ctx context.Context, endp config.Endpoint, starttls bool, tlsConfig *tls.Config) (didTLS bool, err error) { + didTLS, cl, conn, err := c.attemptConnect(ctx, true, endp, starttls, tlsConfig) + if err != nil { + return false, c.wrapClientErr(err, endp.Host) + } + + c.serverName = endp.Host + c.cl = cl + c.conn = conn + + c.Log.DebugMsg("connected", "remote_server", c.serverName, + "local_addr", c.LocalAddr(), "remote_addr", c.RemoteAddr()) + + return didTLS, nil +} + +// TLSError is returned by Connect to indicate the error during STARTTLS +// command execution. +// +// If the endpoint uses Implicit TLS, TLS errors are threated as connection +// errors and thus are not returned as TLSError. +type TLSError struct { + Err error +} + +func (err TLSError) Error() string { + return "smtpconn: " + err.Err.Error() +} + +func (err TLSError) Unwrap() error { + return err.Err +} + +func (c *C) LocalAddr() net.Addr { + if c.conn == nil { + return nil + } + return c.conn.LocalAddr() +} + +func (c *C) RemoteAddr() net.Addr { + if c.conn == nil { + return nil + } + return c.conn.RemoteAddr() +} + +func (c *C) attemptConnect(ctx context.Context, lmtp bool, endp config.Endpoint, starttls bool, tlsConfig *tls.Config) (didTLS bool, cl *smtp.Client, conn net.Conn, err error) { + dialCtx, cancel := context.WithTimeout(ctx, c.ConnectTimeout) + conn, err = c.Dialer(dialCtx, endp.Network(), endp.Address()) + cancel() + if err != nil { + return false, nil, nil, err + } + + if endp.IsTLS() { + cfg := tlsConfig.Clone() + cfg.ServerName = endp.Host + conn = tls.Client(conn, cfg) + } + + c.lmtp = lmtp + // This uses initial greeting timeout of 5 minutes (hardcoded). + if lmtp { + cl = smtp.NewClientLMTP(conn) + } else { + cl = smtp.NewClient(conn) + } + + cl.CommandTimeout = c.CommandTimeout + cl.SubmissionTimeout = c.SubmissionTimeout + + // i18n: hostname is already expected to be in A-labels form. + if err := cl.Hello(c.Hostname); err != nil { + cl.Close() + return false, nil, nil, err + } + + if !starttls { + return false, cl, conn, nil + } + + if ok, _ := cl.Extension("STARTTLS"); !ok { + if err := cl.Quit(); err != nil { + cl.Close() + } + return false, nil, nil, fmt.Errorf("TLS required but unsupported by downstream") + } + + cfg := tlsConfig.Clone() + cfg.ServerName = endp.Host + if err := cl.StartTLS(cfg); err != nil { + // After the handshake failure, the connection may be in a bad state. + // We attempt to send the proper QUIT command though, in case the error happened + // *after* the handshake (e.g. PKI verification fail), we don't log the error in + // this case though. + if err := cl.Quit(); err != nil { + cl.Close() + } + + return false, nil, nil, TLSError{err} + } + + // Re-do HELO using our hostname instead of localhost. + if err := cl.Hello(c.Hostname); err != nil { + cl.Close() + + var tlsErr *tls.CertificateVerificationError + if errors.As(err, &tlsErr) { + return false, nil, nil, TLSError{Err: tlsErr} + } + + return false, nil, nil, err + } + + return true, cl, conn, nil +} + +// Mail sends the MAIL FROM command to the remote server. +// +// SIZE and REQUIRETLS options are forwarded to the remote server as-is. +// SMTPUTF8 is forwarded if supported by the remote server, if it is not +// supported - attempt will be done to convert addresses to the ASCII form, if +// this is not possible, the corresponding method (Mail or Rcpt) will fail. +func (c *C) Mail(ctx context.Context, from string, opts smtp.MailOptions) error { + defer trace.StartRegion(ctx, "smtpconn/MAIL FROM").End() + + outOpts := smtp.MailOptions{ + // Future extensions may add additional fields that should not be + // copied blindly. So we copy only fields we know should be handled + // this way. + + Size: opts.Size, + RequireTLS: opts.RequireTLS, + } + + // INTERNATIONALIZATION: Use SMTPUTF8 is possible, attempt to convert addresses otherwise. + + // There is no way we can accept a message with non-ASCII addresses without SMTPUTF8 + // this is enforced by endpoint/smtp. + if opts.UTF8 { + if ok, _ := c.cl.Extension("SMTPUTF8"); ok { + outOpts.UTF8 = true + } else { + var err error + from, err = address.ToASCII(from) + if err != nil { + return &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 6, 7}, + Message: "SMTPUTF8 is unsupported, cannot convert sender address", + Misc: map[string]interface{}{ + "remote_server": c.serverName, + }, + Err: err, + } + } + } + } + + if err := c.cl.Mail(from, &outOpts); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + return nil +} + +// Rcpts returns the list of recipients that were accepted by the remote server. +func (c *C) Rcpts() []string { + return c.rcpts +} + +func (c *C) ServerName() string { + return c.serverName +} + +func (c *C) Client() *smtp.Client { + return c.cl +} + +func (c *C) IsLMTP() bool { + return c.lmtp +} + +// Rcpt sends the RCPT TO command to the remote server. +// +// If the address is non-ASCII and cannot be converted to ASCII and the remote +// server does not support SMTPUTF8, error will be returned. +func (c *C) Rcpt(ctx context.Context, to string, opts smtp.RcptOptions) error { + defer trace.StartRegion(ctx, "smtpconn/RCPT TO").End() + + outOpts := &smtp.RcptOptions{ + // TODO: DSN support + } + + // If necessary, the extension flag is enabled in Start. + if ok, _ := c.cl.Extension("SMTPUTF8"); !address.IsASCII(to) && !ok { + var err error + to, err = address.ToASCII(to) + if err != nil { + return &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 6, 7}, + Message: "SMTPUTF8 is unsupported, cannot convert recipient address", + Misc: map[string]interface{}{ + "remote_server": c.serverName, + }, + Err: err, + } + } + } + + if err := c.cl.Rcpt(to, outOpts); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + c.rcpts = append(c.rcpts, to) + + return nil +} + +type lmtpError map[string]*smtp.SMTPError + +func (l lmtpError) SetStatus(rcptTo string, err *smtp.SMTPError) { + l[rcptTo] = err +} + +func (l lmtpError) singleError() *smtp.SMTPError { + nonNils := 0 + for _, e := range l { + if e != nil { + nonNils++ + } + } + if nonNils == 1 { + for _, err := range l { + if err != nil { + return err + } + } + } + return nil +} + +func (l lmtpError) Unwrap() error { + if err := l.singleError(); err != nil { + return err + } + return nil +} + +func (l lmtpError) Error() string { + if err := l.singleError(); err != nil { + return err.Error() + } + return fmt.Sprintf("multiple errors reported by LMTP downstream: %v", map[string]*smtp.SMTPError(l)) +} + +func (c *C) smtpToLMTPData(ctx context.Context, hdr textproto.Header, body io.Reader) error { + statusCb := lmtpError{} + if err := c.LMTPData(ctx, hdr, body, statusCb.SetStatus); err != nil { + return err + } + hasAnyFailures := false + for _, err := range statusCb { + if err != nil { + hasAnyFailures = true + } + } + if hasAnyFailures { + return statusCb + } + return nil +} + +// Data sends the DATA command to the remote server and then sends the message header +// and body. +// +// If the Data command fails, the connection may be in a unclean state (e.g. in +// the middle of message data stream). It is not safe to continue using it. +func (c *C) Data(ctx context.Context, hdr textproto.Header, body io.Reader) error { + defer trace.StartRegion(ctx, "smtpconn/DATA").End() + + if c.IsLMTP() { + return c.smtpToLMTPData(ctx, hdr, body) + } + + wc, err := c.cl.Data() + if err != nil { + return c.wrapClientErr(err, c.serverName) + } + + if err := textproto.WriteHeader(wc, hdr); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + if _, err := io.Copy(wc, body); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + if err := wc.Close(); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + return nil +} + +func (c *C) LMTPData(ctx context.Context, hdr textproto.Header, body io.Reader, statusCb func(string, *smtp.SMTPError)) error { + defer trace.StartRegion(ctx, "smtpconn/LMTPDATA").End() + + wc, err := c.cl.LMTPData(statusCb) + if err != nil { + return c.wrapClientErr(err, c.serverName) + } + + if err := textproto.WriteHeader(wc, hdr); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + if _, err := io.Copy(wc, body); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + if err := wc.Close(); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + return nil +} + +func (c *C) Noop() error { + if c.cl == nil { + return errors.New("smtpconn: not connected") + } + + return c.cl.Noop() +} + +// Close sends the QUIT command, if it fails - it directly closes the +// connection. +func (c *C) Close() error { + c.cl.CommandTimeout = 5 * time.Second + + if err := c.cl.Quit(); err != nil { + var smtpErr *smtp.SMTPError + var netErr *net.OpError + if errors.As(err, &smtpErr) && smtpErr.Code == 421 { + // 421 "Service not available" is typically sent + // when idle timeout happens. + c.Log.DebugMsg("QUIT error", "reason", c.wrapClientErr(err, c.serverName)) + } else if errors.As(err, &netErr) && + (netErr.Timeout() || netErr.Err.Error() == "write: broken pipe" || netErr.Err.Error() == "read: connection reset") { + // The case for silently closed connections. + c.Log.DebugMsg("QUIT error", "reason", c.wrapClientErr(err, c.serverName)) + } else { + c.Log.Error("QUIT error", c.wrapClientErr(err, c.serverName)) + } + + return c.cl.Close() + } + + c.cl = nil + c.serverName = "" + + return nil +} + +// DirectClose closes the underlying connection without sending the QUIT +// command. +func (c *C) DirectClose() error { + c.cl.Close() + c.cl = nil + c.serverName = "" + return nil +} diff --git a/internal/smtpconn/smtpconn_test.go b/internal/smtpconn/smtpconn_test.go new file mode 100644 index 0000000..b8fd647 --- /dev/null +++ b/internal/smtpconn/smtpconn_test.go @@ -0,0 +1,41 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtpconn + +import ( + "flag" + "math/rand" + "os" + "strconv" + "testing" +) + +var testPort string + +func TestMain(m *testing.M) { + remoteSmtpPort := flag.String("test.smtpport", "random", "(maddy) SMTP port to use for connections in tests") + flag.Parse() + + if *remoteSmtpPort == "random" { + *remoteSmtpPort = strconv.Itoa(rand.Intn(65536-10000) + 10000) + } + + testPort = *remoteSmtpPort + os.Exit(m.Run()) +} diff --git a/internal/smtpconn/smtputf8_test.go b/internal/smtpconn/smtputf8_test.go new file mode 100644 index 0000000..dc580d9 --- /dev/null +++ b/internal/smtpconn/smtputf8_test.go @@ -0,0 +1,165 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtpconn + +import ( + "context" + "strings" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/internal/testutils" +) + +func doTestDelivery(t *testing.T, conn *C, from string, to []string, opts smtp.MailOptions) error { + t.Helper() + + if err := conn.Mail(context.Background(), from, opts); err != nil { + return err + } + for _, rcpt := range to { + if err := conn.Rcpt(context.Background(), rcpt, smtp.RcptOptions{}); err != nil { + return err + } + } + + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + return conn.Data(context.Background(), hdr, strings.NewReader("foobar\n")) +} + +func TestSMTPUTF8(t *testing.T) { + type test struct { + clientSender string + clientRcpt string + + serverUTF8 bool + serverSender string + serverRcpt string + + expectUTF8 bool + expectErr *exterrors.SMTPError + } + check := func(case_ test) { + t.Helper() + + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + srv.EnableSMTPUTF8 = case_.serverUTF8 + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + c := New() + c.Log = testutils.Logger(t, "target.smtp") + if _, err := c.Connect(context.Background(), config.Endpoint{ + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, false, nil); err != nil { + t.Fatal(err) + } + defer c.Close() + + err := doTestDelivery(t, c, case_.clientSender, []string{case_.clientRcpt}, + smtp.MailOptions{UTF8: true}) + if err != nil { + if case_.expectErr == nil { + t.Error("Unexpected failure") + } else { + testutils.CheckSMTPErr(t, err, case_.expectErr.Code, case_.expectErr.EnhancedCode, case_.expectErr.Message) + } + return + } else if case_.expectErr != nil { + t.Error("Unexpected success") + } + + be.CheckMsg(t, 0, case_.serverSender, []string{case_.serverRcpt}) + if be.Messages[0].Opts.UTF8 != case_.expectUTF8 { + t.Errorf("expectUTF8 = %v, SMTPUTF8 = %v", case_.expectErr, be.Messages[0].Opts.UTF8) + } + } + + check(test{ + clientSender: "test@тест.example.org", + clientRcpt: "test@example.invalid", + serverSender: "test@xn--e1aybc.example.org", + serverRcpt: "test@example.invalid", + }) + check(test{ + clientSender: "test@example.org", + clientRcpt: "test@тест.example.invalid", + serverSender: "test@example.org", + serverRcpt: "test@xn--e1aybc.example.invalid", + }) + check(test{ + clientSender: "тест@example.org", + clientRcpt: "test@example.invalid", + serverUTF8: false, + expectErr: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 6, 7}, + Message: "SMTPUTF8 is unsupported, cannot convert sender address", + }, + }) + check(test{ + clientSender: "test@example.org", + clientRcpt: "тест@example.invalid", + serverUTF8: false, + expectErr: &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 6, 7}, + Message: "SMTPUTF8 is unsupported, cannot convert recipient address", + }, + }) + check(test{ + clientSender: "test@тест.org", + clientRcpt: "test@example.invalid", + serverSender: "test@тест.org", + serverRcpt: "test@example.invalid", + serverUTF8: true, + expectUTF8: true, + }) + check(test{ + clientSender: "test@example.org", + clientRcpt: "test@тест.example.invalid", + serverSender: "test@example.org", + serverRcpt: "test@тест.example.invalid", + serverUTF8: true, + expectUTF8: true, + }) + check(test{ + clientSender: "тест@example.org", + clientRcpt: "test@example.invalid", + serverSender: "тест@example.org", + serverRcpt: "test@example.invalid", + serverUTF8: true, + expectUTF8: true, + }) + check(test{ + clientSender: "test@example.org", + clientRcpt: "тест@example.invalid", + serverSender: "test@example.org", + serverRcpt: "тест@example.invalid", + serverUTF8: true, + expectUTF8: true, + }) +} diff --git a/internal/storage/blob/fs/fs.go b/internal/storage/blob/fs/fs.go new file mode 100644 index 0000000..e8c9b38 --- /dev/null +++ b/internal/storage/blob/fs/fs.go @@ -0,0 +1,95 @@ +package fs + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +// FSStore struct represents directory on FS used to store blobs. +type FSStore struct { + instName string + root string +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + switch len(inlineArgs) { + case 0: + return &FSStore{instName: instName}, nil + case 1: + return &FSStore{instName: instName, root: inlineArgs[0]}, nil + default: + return nil, fmt.Errorf("storage.blob.fs: 1 or 0 arguments expected") + } +} + +func (s FSStore) Name() string { + return "storage.blob.fs" +} + +func (s FSStore) InstanceName() string { + return s.instName +} + +func (s *FSStore) Init(cfg *config.Map) error { + cfg.String("root", false, false, s.root, &s.root) + if _, err := cfg.Process(); err != nil { + return err + } + + if s.root == "" { + return config.NodeErr(cfg.Block, "storage.blob.fs: directory not set") + } + + if err := os.MkdirAll(s.root, os.ModeDir|os.ModePerm); err != nil { + return err + } + + return nil +} + +func (s *FSStore) Open(_ context.Context, key string) (io.ReadCloser, error) { + f, err := os.Open(filepath.Join(s.root, key)) + if err != nil { + if os.IsNotExist(err) { + return nil, module.ErrNoSuchBlob + } + return nil, err + } + return f, nil +} + +func (s *FSStore) Create(_ context.Context, key string, blobSize int64) (module.Blob, error) { + f, err := os.Create(filepath.Join(s.root, key)) + if err != nil { + return nil, err + } + if blobSize >= 0 { + if err := f.Truncate(blobSize); err != nil { + return nil, err + } + } + return f, nil +} + +func (s *FSStore) Delete(_ context.Context, keys []string) error { + for _, key := range keys { + if err := os.Remove(filepath.Join(s.root, key)); err != nil { + if os.IsNotExist(err) { + continue + } + return err + } + } + return nil +} + +func init() { + var _ module.BlobStore = &FSStore{} + module.Register(FSStore{}.Name(), New) +} diff --git a/internal/storage/blob/fs/fs_test.go b/internal/storage/blob/fs/fs_test.go new file mode 100644 index 0000000..2c8f766 --- /dev/null +++ b/internal/storage/blob/fs/fs_test.go @@ -0,0 +1,19 @@ +package fs + +import ( + "os" + "testing" + + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/storage/blob" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestFS(t *testing.T) { + blob.TestStore(t, func() module.BlobStore { + dir := testutils.Dir(t) + return &FSStore{instName: "test", root: dir} + }, func(store module.BlobStore) { + os.RemoveAll(store.(*FSStore).root) + }) +} diff --git a/internal/storage/blob/s3/s3.go b/internal/storage/blob/s3/s3.go new file mode 100644 index 0000000..af01d88 --- /dev/null +++ b/internal/storage/blob/s3/s3.go @@ -0,0 +1,196 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "net/http" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +const modName = "storage.blob.s3" + +const ( + credsTypeFileMinio = "file_minio" + credsTypeFileAWS = "file_aws" + credsTypeAccessKey = "access_key" + credsTypeIAM = "iam" + credsTypeDefault = credsTypeAccessKey +) + +type Store struct { + instName string + log log.Logger + + endpoint string + cl *minio.Client + + bucketName string + objectPrefix string +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, fmt.Errorf("%s: expected 0 arguments", modName) + } + + return &Store{ + instName: instName, + log: log.Logger{Name: modName}, + }, nil +} + +func (s *Store) Init(cfg *config.Map) error { + var ( + secure bool + accessKeyID string + secretAccessKey string + credsType string + location string + ) + cfg.String("endpoint", false, true, "", &s.endpoint) + cfg.Bool("secure", false, true, &secure) + cfg.String("access_key", false, true, "", &accessKeyID) + cfg.String("secret_key", false, true, "", &secretAccessKey) + cfg.String("bucket", false, true, "", &s.bucketName) + cfg.String("region", false, false, "", &location) + cfg.String("object_prefix", false, false, "", &s.objectPrefix) + cfg.String("creds", false, false, credsTypeDefault, &credsType) + + if _, err := cfg.Process(); err != nil { + return err + } + if s.endpoint == "" { + return fmt.Errorf("%s: endpoint not set", modName) + } + + var creds *credentials.Credentials + + switch credsType { + case credsTypeFileMinio: + creds = credentials.NewFileMinioClient("", "") + case credsTypeFileAWS: + creds = credentials.NewFileAWSCredentials("", "") + case credsTypeIAM: + creds = credentials.NewIAM("") + case credsTypeAccessKey: + creds = credentials.NewStaticV4(accessKeyID, secretAccessKey, "") + default: + creds = credentials.NewStaticV4(accessKeyID, secretAccessKey, "") + } + + cl, err := minio.New(s.endpoint, &minio.Options{ + Creds: creds, + Secure: secure, + Region: location, + }) + if err != nil { + return fmt.Errorf("%s: %w", modName, err) + } + + s.cl = cl + return nil +} + +func (s *Store) Name() string { + return modName +} + +func (s *Store) InstanceName() string { + return s.instName +} + +type s3blob struct { + pw *io.PipeWriter + didSync bool + errCh chan error +} + +func (b *s3blob) Sync() error { + // We do this in Sync instead of Close because + // backend may not actually check the error of Close. + // The problematic restriction is that Sync can now be called + // only once. + if b.didSync { + panic("storage.blob.s3: Sync called twice for a blob object") + } + + b.pw.Close() + b.didSync = true + return <-b.errCh +} + +func (b *s3blob) Write(p []byte) (n int, err error) { + return b.pw.Write(p) +} + +func (b *s3blob) Close() error { + if !b.didSync { + if err := b.pw.CloseWithError(fmt.Errorf("storage.blob.s3: blob closed without Sync")); err != nil { + panic(err) + } + } + return nil +} + +func (s *Store) Create(ctx context.Context, key string, blobSize int64) (module.Blob, error) { + pr, pw := io.Pipe() + errCh := make(chan error, 1) + + go func() { + partSize := uint64(0) + if blobSize == module.UnknownBlobSize { + // Without this, minio-go will allocate 500 MiB buffer which + // is a little too much. + // https://github.com/minio/minio-go/issues/1478 + partSize = 1 * 1024 * 1024 /* 1 MiB */ + } + _, err := s.cl.PutObject(ctx, s.bucketName, s.objectPrefix+key, pr, blobSize, minio.PutObjectOptions{ + PartSize: partSize, + }) + if err != nil { + if err := pr.CloseWithError(fmt.Errorf("s3 PutObject: %w", err)); err != nil { + panic(err) + } + } + errCh <- err + }() + + return &s3blob{ + pw: pw, + errCh: errCh, + }, nil +} + +func (s *Store) Open(ctx context.Context, key string) (io.ReadCloser, error) { + obj, err := s.cl.GetObject(ctx, s.bucketName, s.objectPrefix+key, minio.GetObjectOptions{}) + if err != nil { + resp := minio.ToErrorResponse(err) + if resp.StatusCode == http.StatusNotFound { + return nil, module.ErrNoSuchBlob + } + return nil, err + } + return obj, nil +} + +func (s *Store) Delete(ctx context.Context, keys []string) error { + var lastErr error + for _, k := range keys { + lastErr = s.cl.RemoveObject(ctx, s.bucketName, s.objectPrefix+k, minio.RemoveObjectOptions{}) + if lastErr != nil { + s.log.Error("failed to delete object", lastErr, s.objectPrefix+k) + } + } + return lastErr +} + +func init() { + var _ module.BlobStore = &Store{} + module.Register(modName, New) +} diff --git a/internal/storage/blob/s3/s3_test.go b/internal/storage/blob/s3/s3_test.go new file mode 100644 index 0000000..98dd228 --- /dev/null +++ b/internal/storage/blob/s3/s3_test.go @@ -0,0 +1,71 @@ +package s3 + +import ( + "net/http/httptest" + "testing" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/storage/blob" + "github.com/johannesboyne/gofakes3" + "github.com/johannesboyne/gofakes3/backend/s3mem" +) + +func TestFS(t *testing.T) { + var ( + backend gofakes3.Backend + faker *gofakes3.GoFakeS3 + ts *httptest.Server + ) + + blob.TestStore(t, func() module.BlobStore { + backend = s3mem.New() + faker = gofakes3.New(backend) + ts = httptest.NewServer(faker.Server()) + + if err := backend.CreateBucket("maddy-test"); err != nil { + panic(err) + } + + st := &Store{instName: "test"} + err := st.Init(config.NewMap(map[string]interface{}{}, config.Node{ + Children: []config.Node{ + { + Name: "endpoint", + Args: []string{ts.Listener.Addr().String()}, + }, + { + Name: "secure", + Args: []string{"false"}, + }, + { + Name: "access_key", + Args: []string{"access-key"}, + }, + { + Name: "secret_key", + Args: []string{"secret-key"}, + }, + { + Name: "bucket", + Args: []string{"maddy-test"}, + }, + }, + })) + if err != nil { + panic(err) + } + + return st + }, func(store module.BlobStore) { + ts.Close() + + backend = s3mem.New() + faker = gofakes3.New(backend) + ts = httptest.NewServer(faker.Server()) + }) + + if ts != nil { + ts.Close() + } +} diff --git a/internal/storage/blob/test_blob.go b/internal/storage/blob/test_blob.go new file mode 100644 index 0000000..9672efe --- /dev/null +++ b/internal/storage/blob/test_blob.go @@ -0,0 +1,62 @@ +//go:build cgo && !no_sqlite3 +// +build cgo,!no_sqlite3 + +package blob + +import ( + "math/rand" + "testing" + + backendtests "github.com/foxcpp/go-imap-backend-tests" + imapsql "github.com/foxcpp/go-imap-sql" + "github.com/foxcpp/maddy/framework/module" + imapsql2 "github.com/foxcpp/maddy/internal/storage/imapsql" + "github.com/foxcpp/maddy/internal/testutils" +) + +type testBack struct { + backendtests.Backend + ExtStore module.BlobStore +} + +func TestStore(t *testing.T, newStore func() module.BlobStore, cleanStore func(module.BlobStore)) { + // We use go-imap-sql backend and run a subset of + // go-imap-backend-tests related to loading and saving messages. + // + // In the future we should probably switch to using a memory + // backend for this. + + backendtests.Whitelist = []string{ + t.Name() + "/Mailbox_CreateMessage", + t.Name() + "/Mailbox_ListMessages_Body", + t.Name() + "/Mailbox_CopyMessages", + t.Name() + "/Mailbox_Expunge", + t.Name() + "/Mailbox_MoveMessages", + } + + initBackend := func() backendtests.Backend { + randSrc := rand.NewSource(0) + prng := rand.New(randSrc) + store := newStore() + + b, err := imapsql.New("sqlite3", ":memory:", + imapsql2.ExtBlobStore{Base: store}, imapsql.Opts{ + PRNG: prng, + Log: testutils.Logger(t, "imapsql"), + }, + ) + if err != nil { + panic(err) + } + return testBack{Backend: b, ExtStore: store} + } + cleanBackend := func(bi backendtests.Backend) { + b := bi.(testBack) + if err := b.Backend.(*imapsql.Backend).Close(); err != nil { + panic(err) + } + cleanStore(b.ExtStore) + } + + backendtests.RunTests(t, initBackend, cleanBackend) +} diff --git a/internal/storage/blob/test_blob_nosqlite.go b/internal/storage/blob/test_blob_nosqlite.go new file mode 100644 index 0000000..601f467 --- /dev/null +++ b/internal/storage/blob/test_blob_nosqlite.go @@ -0,0 +1,14 @@ +//go:build !cgo || no_sqlite3 +// +build !cgo no_sqlite3 + +package blob + +import ( + "testing" + + "github.com/foxcpp/maddy/framework/module" +) + +func TestStore(t *testing.T, newStore func() module.BlobStore, cleanStore func(module.BlobStore)) { + t.Skip("storage.blob tests require CGo and sqlite3") +} diff --git a/internal/storage/imapsql/bench_test.go b/internal/storage/imapsql/bench_test.go new file mode 100644 index 0000000..04cfac9 --- /dev/null +++ b/internal/storage/imapsql/bench_test.go @@ -0,0 +1,90 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imapsql + +import ( + "flag" + "strconv" + "testing" + "time" + + imapsql "github.com/foxcpp/go-imap-sql" + "github.com/foxcpp/maddy/internal/testutils" +) + +var ( + testDB string + testDSN string + testFsstore string +) + +func init() { + flag.StringVar(&testDB, "sql.testdb", "", "Database to use for storage/sql benchmarks") + flag.StringVar(&testDSN, "sql.testdsn", "", "DSN to use for storage/sql benchmarks") + flag.StringVar(&testFsstore, "sql.testfsstore", "", "fsstore location to use for storage/sql benchmarks") +} + +func createTestDB(tb testing.TB, compAlgo string) *Storage { + if testDB == "" || testDSN == "" || testFsstore == "" { + tb.Skip("-sql.testdb, -sql.testdsn and -sql.testfsstore should be specified to run this benchmark") + } + + db, err := imapsql.New(testDB, testDSN, &imapsql.FSStore{Root: testFsstore}, imapsql.Opts{ + CompressAlgo: compAlgo, + }) + if err != nil { + tb.Fatal(err) + } + return &Storage{ + Back: db, + } +} + +func BenchmarkStorage_Delivery(b *testing.B) { + randomKey := "rcpt-" + strconv.FormatInt(time.Now().UnixNano(), 10) + "@example.org" + + be := createTestDB(b, "") + if err := be.CreateIMAPAcct(randomKey); err != nil { + b.Fatal(err) + } + + testutils.BenchDelivery(b, be, "sender@example.org", []string{randomKey}) +} + +func BenchmarkStorage_DeliveryLZ4(b *testing.B) { + randomKey := "rcpt-" + strconv.FormatInt(time.Now().UnixNano(), 10) + "@example.org" + + be := createTestDB(b, "lz4") + if err := be.CreateIMAPAcct(randomKey); err != nil { + b.Fatal(err) + } + + testutils.BenchDelivery(b, be, "sender@example.org", []string{randomKey}) +} + +func BenchmarkStorage_DeliveryZstd(b *testing.B) { + randomKey := "rcpt-" + strconv.FormatInt(time.Now().UnixNano(), 10) + "@example.org" + + be := createTestDB(b, "zstd") + if err := be.CreateIMAPAcct(randomKey); err != nil { + b.Fatal(err) + } + + testutils.BenchDelivery(b, be, "sender@example.org", []string{randomKey}) +} diff --git a/internal/storage/imapsql/delivery.go b/internal/storage/imapsql/delivery.go new file mode 100644 index 0000000..60cb2e1 --- /dev/null +++ b/internal/storage/imapsql/delivery.go @@ -0,0 +1,168 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imapsql + +import ( + "context" + "runtime/trace" + + "github.com/emersion/go-imap" + "github.com/emersion/go-imap/backend" + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + imapsql "github.com/foxcpp/go-imap-sql" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +type addedRcpt struct { + rcptTo string +} +type delivery struct { + store *Storage + msgMeta *module.MsgMetadata + d imapsql.Delivery + mailFrom string + + addedRcpts map[string]addedRcpt +} + +func (d *delivery) String() string { + return d.store.Name() + ":" + d.store.InstanceName() +} + +func userDoesNotExist(actual error) error { + return &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 1}, + Message: "User does not exist", + TargetName: "imapsql", + Err: actual, + } +} + +func (d *delivery) AddRcpt(ctx context.Context, rcptTo string, _ smtp.RcptOptions) error { + defer trace.StartRegion(ctx, "sql/AddRcpt").End() + + accountName, err := d.store.deliveryNormalize(ctx, rcptTo) + if err != nil { + return userDoesNotExist(err) + } + + if _, ok := d.addedRcpts[accountName]; ok { + return nil + } + + // This header is added to the message only for that recipient. + // go-imap-sql does certain optimizations to store the message + // with small amount of per-recipient data in a efficient way. + userHeader := textproto.Header{} + userHeader.Add("Delivered-To", accountName) + + if err := d.d.AddRcpt(accountName, userHeader); err != nil { + if err == imapsql.ErrUserDoesntExists || err == backend.ErrNoSuchMailbox { + return userDoesNotExist(err) + } + if _, ok := err.(imapsql.SerializationError); ok { + return &exterrors.SMTPError{ + Code: 453, + EnhancedCode: exterrors.EnhancedCode{4, 3, 2}, + Message: "Internal server error, try again later", + TargetName: "imapsql", + Err: err, + } + } + return err + } + + d.addedRcpts[accountName] = addedRcpt{ + rcptTo: rcptTo, + } + return nil +} + +func (d *delivery) Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error { + defer trace.StartRegion(ctx, "sql/Body").End() + + if !d.msgMeta.Quarantine && d.store.filters != nil { + for rcpt, rcptData := range d.addedRcpts { + folder, flags, err := d.store.filters.IMAPFilter(rcpt, rcptData.rcptTo, d.msgMeta, header, body) + if err != nil { + d.store.Log.Error("IMAPFilter failed", err, "rcpt", rcpt) + continue + } + d.d.UserMailbox(rcpt, folder, flags) + } + } + + if d.msgMeta.Quarantine { + if err := d.d.SpecialMailbox(imap.JunkAttr, d.store.junkMbox); err != nil { + if _, ok := err.(imapsql.SerializationError); ok { + return &exterrors.SMTPError{ + Code: 453, + EnhancedCode: exterrors.EnhancedCode{4, 3, 2}, + Message: "Storage access serialiation problem, try again later", + TargetName: "imapsql", + Err: err, + } + } + return err + } + } + + header = header.Copy() + header.Add("Return-Path", "<"+target.SanitizeForHeader(d.mailFrom)+">") + err := d.d.BodyParsed(header, body.Len(), body) + if _, ok := err.(imapsql.SerializationError); ok { + return &exterrors.SMTPError{ + Code: 453, + EnhancedCode: exterrors.EnhancedCode{4, 3, 2}, + Message: "Storage access serialiation problem, try again later", + TargetName: "imapsql", + Err: err, + } + } + return err +} + +func (d *delivery) Abort(ctx context.Context) error { + defer trace.StartRegion(ctx, "sql/Abort").End() + + return d.d.Abort() +} + +func (d *delivery) Commit(ctx context.Context) error { + defer trace.StartRegion(ctx, "sql/Commit").End() + + return d.d.Commit() +} + +func (store *Storage) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + defer trace.StartRegion(ctx, "sql/Start").End() + + return &delivery{ + store: store, + msgMeta: msgMeta, + mailFrom: mailFrom, + d: store.Back.NewDelivery(), + addedRcpts: map[string]addedRcpt{}, + }, nil +} diff --git a/internal/storage/imapsql/external_blob_store.go b/internal/storage/imapsql/external_blob_store.go new file mode 100644 index 0000000..0fb9cd7 --- /dev/null +++ b/internal/storage/imapsql/external_blob_store.go @@ -0,0 +1,68 @@ +package imapsql + +import ( + "context" + "io" + + imapsql "github.com/foxcpp/go-imap-sql" + "github.com/foxcpp/maddy/framework/module" +) + +type ExtBlob struct { + io.ReadCloser +} + +func (e ExtBlob) Sync() error { + panic("not implemented") +} + +func (e ExtBlob) Write(p []byte) (n int, err error) { + panic("not implemented") +} + +type WriteExtBlob struct { + module.Blob +} + +func (w WriteExtBlob) Read(p []byte) (n int, err error) { + panic("not implemented") +} + +type ExtBlobStore struct { + Base module.BlobStore +} + +func (e ExtBlobStore) Create(key string, objSize int64) (imapsql.ExtStoreObj, error) { + blob, err := e.Base.Create(context.TODO(), key, objSize) + if err != nil { + return nil, imapsql.ExternalError{ + NonExistent: err == module.ErrNoSuchBlob, + Key: key, + Err: err, + } + } + return WriteExtBlob{Blob: blob}, nil +} + +func (e ExtBlobStore) Open(key string) (imapsql.ExtStoreObj, error) { + blob, err := e.Base.Open(context.TODO(), key) + if err != nil { + return nil, imapsql.ExternalError{ + NonExistent: err == module.ErrNoSuchBlob, + Key: key, + Err: err, + } + } + return ExtBlob{ReadCloser: blob}, nil +} + +func (e ExtBlobStore) Delete(keys []string) error { + err := e.Base.Delete(context.TODO(), keys) + if err != nil { + return imapsql.ExternalError{ + Key: "", + Err: err, + } + } + return nil +} diff --git a/internal/storage/imapsql/imapsql.go b/internal/storage/imapsql/imapsql.go new file mode 100644 index 0000000..711a34e --- /dev/null +++ b/internal/storage/imapsql/imapsql.go @@ -0,0 +1,438 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package imapsql implements SQL-based storage module +// using go-imap-sql library (github.com/foxcpp/go-imap-sql). +// +// Interfaces implemented: +// - module.StorageBackend +// - module.PlainAuth +// - module.DeliveryTarget +package imapsql + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "path/filepath" + "runtime/debug" + "strconv" + "strings" + + "github.com/emersion/go-imap" + sortthread "github.com/emersion/go-imap-sortthread" + "github.com/emersion/go-imap/backend" + mess "github.com/foxcpp/go-imap-mess" + imapsql "github.com/foxcpp/go-imap-sql" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/authz" + "github.com/foxcpp/maddy/internal/updatepipe" + "github.com/foxcpp/maddy/internal/updatepipe/pubsub" + + _ "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" +) + +type Storage struct { + Back *imapsql.Backend + instName string + Log log.Logger + + junkMbox string + + driver string + dsn []string + + resolver dns.Resolver + + updPipe updatepipe.P + updPushStop chan struct{} + outboundUpds chan mess.Update + + filters module.IMAPFilter + + deliveryMap module.Table + deliveryNormalize func(context.Context, string) (string, error) + authMap module.Table + authNormalize func(context.Context, string) (string, error) +} + +func (store *Storage) Name() string { + return "imapsql" +} + +func (store *Storage) InstanceName() string { + return store.instName +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + store := &Storage{ + instName: instName, + Log: log.Logger{Name: "imapsql"}, + resolver: dns.DefaultResolver(), + } + if len(inlineArgs) != 0 { + if len(inlineArgs) == 1 { + return nil, errors.New("imapsql: expected at least 2 arguments") + } + + store.driver = inlineArgs[0] + store.dsn = inlineArgs[1:] + } + return store, nil +} + +func (store *Storage) Init(cfg *config.Map) error { + var ( + driver string + dsn []string + appendlimitVal int64 = -1 + compression []string + authNormalize string + deliveryNormalize string + + blobStore module.BlobStore + ) + + opts := imapsql.Opts{} + cfg.String("driver", false, false, store.driver, &driver) + cfg.StringList("dsn", false, false, store.dsn, &dsn) + cfg.Callback("fsstore", func(m *config.Map, node config.Node) error { + store.Log.Msg("'fsstore' directive is deprecated, use 'msg_store fs' instead") + return modconfig.ModuleFromNode("storage.blob", append([]string{"fs"}, node.Args...), + node, m.Globals, &blobStore) + }) + cfg.Custom("msg_store", false, false, func() (interface{}, error) { + var store module.BlobStore + err := modconfig.ModuleFromNode("storage.blob", []string{"fs", "messages"}, + config.Node{}, nil, &store) + return store, err + }, func(m *config.Map, node config.Node) (interface{}, error) { + var store module.BlobStore + err := modconfig.ModuleFromNode("storage.blob", node.Args, + node, m.Globals, &store) + return store, err + }, &blobStore) + cfg.StringList("compression", false, false, []string{"off"}, &compression) + cfg.DataSize("appendlimit", false, false, 32*1024*1024, &appendlimitVal) + cfg.Bool("debug", true, false, &store.Log.Debug) + cfg.Int("sqlite3_cache_size", false, false, 0, &opts.CacheSize) + cfg.Int("sqlite3_busy_timeout", false, false, 5000, &opts.BusyTimeout) + cfg.Bool("disable_recent", false, true, &opts.DisableRecent) + cfg.String("junk_mailbox", false, false, "Junk", &store.junkMbox) + cfg.Custom("imap_filter", false, false, func() (interface{}, error) { + return nil, nil + }, func(m *config.Map, node config.Node) (interface{}, error) { + var filter module.IMAPFilter + err := modconfig.GroupFromNode("imap_filters", node.Args, node, m.Globals, &filter) + return filter, err + }, &store.filters) + cfg.Custom("auth_map", false, false, func() (interface{}, error) { + return nil, nil + }, modconfig.TableDirective, &store.authMap) + cfg.String("auth_normalize", false, false, "auto", &authNormalize) + cfg.Custom("delivery_map", false, false, func() (interface{}, error) { + return nil, nil + }, modconfig.TableDirective, &store.deliveryMap) + cfg.String("delivery_normalize", false, false, "precis_casefold_email", &deliveryNormalize) + + if _, err := cfg.Process(); err != nil { + return err + } + + if dsn == nil { + return errors.New("imapsql: dsn is required") + } + if driver == "" { + return errors.New("imapsql: driver is required") + } + + if driver == "sqlite3" { + if sqliteImpl == "modernc" { + store.Log.Println("using transpiled SQLite (modernc.org/sqlite), this is experimental") + driver = "sqlite" + } else if sqliteImpl == "cgo" { + store.Log.Debugln("using cgo SQLite") + } else if sqliteImpl == "missing" { + return errors.New("imapsql: SQLite is not supported, recompile without no_sqlite3 tag set") + } + } + + deliveryNormFunc, ok := authz.NormalizeFuncs[deliveryNormalize] + if !ok { + return errors.New("imapsql: unknown normalization function: " + deliveryNormalize) + } + store.deliveryNormalize = func(ctx context.Context, s string) (string, error) { + return deliveryNormFunc(s) + } + if store.deliveryMap != nil { + store.deliveryNormalize = func(ctx context.Context, email string) (string, error) { + email, err := deliveryNormFunc(email) + if err != nil { + return "", err + } + mapped, ok, err := store.deliveryMap.Lookup(ctx, email) + if err != nil || !ok { + return "", userDoesNotExist(err) + } + return mapped, nil + } + } + + if authNormalize != "auto" { + store.Log.Msg("auth_normalize in storage.imapsql is deprecated and will be removed in the next release, use storage_map in imap config instead") + } + authNormFunc, ok := authz.NormalizeFuncs[authNormalize] + if !ok { + return errors.New("imapsql: unknown normalization function: " + authNormalize) + } + store.authNormalize = func(ctx context.Context, s string) (string, error) { + return authNormFunc(s) + } + if store.authMap != nil { + store.Log.Msg("auth_map in storage.imapsql is deprecated and will be removed in the next release, use storage_map in imap config instead") + store.authNormalize = func(ctx context.Context, username string) (string, error) { + username, err := authNormFunc(username) + if err != nil { + return "", err + } + mapped, ok, err := store.authMap.Lookup(ctx, username) + if err != nil || !ok { + return "", userDoesNotExist(err) + } + return mapped, nil + } + } + + opts.Log = &store.Log + + if appendlimitVal == -1 { + opts.MaxMsgBytes = nil + } else { + // int is 32-bit on some platforms, so cut off values we can't actually + // use. + if int64(uint32(appendlimitVal)) != appendlimitVal { + return errors.New("imapsql: appendlimit value is too big") + } + opts.MaxMsgBytes = new(uint32) + *opts.MaxMsgBytes = uint32(appendlimitVal) + } + var err error + + dsnStr := strings.Join(dsn, " ") + + if len(compression) != 0 { + switch compression[0] { + case "zstd", "lz4": + opts.CompressAlgo = compression[0] + if len(compression) == 2 { + opts.CompressAlgoParams = compression[1] + if _, err := strconv.Atoi(compression[1]); err != nil { + return errors.New("imapsql: first argument for lz4 and zstd is compression level") + } + } + if len(compression) > 2 { + return errors.New("imapsql: expected at most 2 arguments") + } + case "off": + if len(compression) > 1 { + return errors.New("imapsql: expected at most 1 arguments") + } + default: + return errors.New("imapsql: unknown compression algorithm") + } + } + + store.Back, err = imapsql.New(driver, dsnStr, ExtBlobStore{Base: blobStore}, opts) + if err != nil { + return fmt.Errorf("imapsql: %s", err) + } + + store.Log.Debugln("go-imap-sql version", imapsql.VersionStr) + + store.driver = driver + store.dsn = dsn + + return nil +} + +func (store *Storage) EnableUpdatePipe(mode updatepipe.BackendMode) error { + if store.updPipe != nil { + return nil + } + + switch store.driver { + case "sqlite3": + dbId := sha1.Sum([]byte(strings.Join(store.dsn, " "))) + sockPath := filepath.Join( + config.RuntimeDirectory, + fmt.Sprintf("sql-%s.sock", hex.EncodeToString(dbId[:]))) + store.Log.DebugMsg("using unix socket for external updates", "path", sockPath) + store.updPipe = &updatepipe.UnixSockPipe{ + SockPath: sockPath, + Log: log.Logger{Name: "storage.imapsql/updpipe", Debug: store.Log.Debug}, + } + case "postgres": + store.Log.DebugMsg("using PostgreSQL broker for external updates") + ps, err := pubsub.NewPQ(strings.Join(store.dsn, " ")) + if err != nil { + return fmt.Errorf("enable_update_pipe: %w", err) + } + ps.Log = log.Logger{Name: "storage.imapsql/updpipe/pubsub", Debug: store.Log.Debug} + pipe := &updatepipe.PubSubPipe{ + PubSub: ps, + Log: log.Logger{Name: "storage.imapsql/updpipe", Debug: store.Log.Debug}, + } + store.Back.UpdateManager().ExternalUnsubscribe = pipe.Unsubscribe + store.Back.UpdateManager().ExternalSubscribe = pipe.Subscribe + store.updPipe = pipe + default: + return errors.New("imapsql: driver does not have an update pipe implementation") + } + + inbound := make(chan mess.Update, 32) + outbound := make(chan mess.Update, 10) + store.outboundUpds = outbound + + if mode == updatepipe.ModeReplicate { + if err := store.updPipe.Listen(inbound); err != nil { + store.updPipe = nil + return err + } + } + + if err := store.updPipe.InitPush(); err != nil { + store.updPipe = nil + return err + } + + store.Back.UpdateManager().SetExternalSink(outbound) + + store.updPushStop = make(chan struct{}, 1) + go func() { + defer func() { + // Ensure we sent all outbound updates. + for upd := range outbound { + if err := store.updPipe.Push(upd); err != nil { + store.Log.Error("IMAP update pipe push failed", err) + } + } + store.updPushStop <- struct{}{} + + if err := recover(); err != nil { + stack := debug.Stack() + log.Printf("panic during imapsql update push: %v\n%s", err, stack) + } + }() + + for { + select { + case u := <-inbound: + store.Log.DebugMsg("external update received", "type", u.Type, "key", u.Key) + store.Back.UpdateManager().ExternalUpdate(u) + case u, ok := <-outbound: + if !ok { + return + } + store.Log.DebugMsg("sending external update", "type", u.Type, "key", u.Key) + if err := store.updPipe.Push(u); err != nil { + store.Log.Error("IMAP update pipe push failed", err) + } + } + } + }() + + return nil +} + +func (store *Storage) I18NLevel() int { + return 1 +} + +func (store *Storage) IMAPExtensions() []string { + return []string{"APPENDLIMIT", "MOVE", "CHILDREN", "SPECIAL-USE", "I18NLEVEL=1", "SORT", "THREAD=ORDEREDSUBJECT"} +} + +func (store *Storage) CreateMessageLimit() *uint32 { + return store.Back.CreateMessageLimit() +} + +func (store *Storage) GetOrCreateIMAPAcct(username string) (backend.User, error) { + accountName, err := store.authNormalize(context.TODO(), username) + if err != nil { + return nil, backend.ErrInvalidCredentials + } + + return store.Back.GetOrCreateUser(accountName) +} + +func (store *Storage) Lookup(ctx context.Context, key string) (string, bool, error) { + accountName, err := store.authNormalize(ctx, key) + if err != nil { + return "", false, nil + } + + usr, err := store.Back.GetUser(accountName) + if err != nil { + if errors.Is(err, imapsql.ErrUserDoesntExists) { + return "", false, nil + } + return "", false, err + } + if err := usr.Logout(); err != nil { + store.Log.Error("logout failed", err, "username", accountName) + } + + return "", true, nil +} + +func (store *Storage) Close() error { + // Stop backend from generating new updates. + store.Back.Close() + + // Wait for 'updates replicate' goroutine to actually stop so we will send + // all updates before shutting down (this is especially important for + // maddy subcommands). + if store.updPipe != nil { + close(store.outboundUpds) + <-store.updPushStop + + store.updPipe.Close() + } + + return nil +} + +func (store *Storage) Login(_ *imap.ConnInfo, usenrame, password string) (backend.User, error) { + panic("This method should not be called and is added only to satisfy backend.Backend interface") +} + +func (store *Storage) SupportedThreadAlgorithms() []sortthread.ThreadAlgorithm { + return []sortthread.ThreadAlgorithm{sortthread.OrderedSubject} +} + +func init() { + module.Register("storage.imapsql", New) + module.Register("target.imapsql", New) +} diff --git a/internal/storage/imapsql/maddyctl.go b/internal/storage/imapsql/maddyctl.go new file mode 100644 index 0000000..fa76638 --- /dev/null +++ b/internal/storage/imapsql/maddyctl.go @@ -0,0 +1,42 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imapsql + +import ( + "github.com/emersion/go-imap/backend" +) + +// These methods wrap corresponding go-imap-sql methods, but also apply +// maddy-specific credentials rules. + +func (store *Storage) ListIMAPAccts() ([]string, error) { + return store.Back.ListUsers() +} + +func (store *Storage) CreateIMAPAcct(accountName string) error { + return store.Back.CreateUser(accountName) +} + +func (store *Storage) DeleteIMAPAcct(accountName string) error { + return store.Back.DeleteUser(accountName) +} + +func (store *Storage) GetIMAPAcct(accountName string) (backend.User, error) { + return store.Back.GetUser(accountName) +} diff --git a/internal/storage/imapsql/modernc_sqlite3.go b/internal/storage/imapsql/modernc_sqlite3.go new file mode 100644 index 0000000..696b4c0 --- /dev/null +++ b/internal/storage/imapsql/modernc_sqlite3.go @@ -0,0 +1,26 @@ +//go:build !nosqlite3 && !cgo +// +build !nosqlite3,!cgo + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imapsql + +import _ "modernc.org/sqlite" + +const sqliteImpl = "modernc" diff --git a/internal/storage/imapsql/no_sqlite3.go b/internal/storage/imapsql/no_sqlite3.go new file mode 100644 index 0000000..525f8e4 --- /dev/null +++ b/internal/storage/imapsql/no_sqlite3.go @@ -0,0 +1,24 @@ +//go:build nosqlite3 +// +build nosqlite3 + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imapsql + +const sqliteImpl = "missing" diff --git a/internal/storage/imapsql/sqlite3.go b/internal/storage/imapsql/sqlite3.go new file mode 100644 index 0000000..599f39d --- /dev/null +++ b/internal/storage/imapsql/sqlite3.go @@ -0,0 +1,26 @@ +//go:build !nosqlite3 && cgo +// +build !nosqlite3,cgo + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imapsql + +import _ "github.com/mattn/go-sqlite3" + +const sqliteImpl = "cgo" diff --git a/internal/table/chain.go b/internal/table/chain.go new file mode 100644 index 0000000..72eceba --- /dev/null +++ b/internal/table/chain.go @@ -0,0 +1,131 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/module" +) + +type Chain struct { + modName string + instName string + + chain []module.Table + optional []bool +} + +func NewChain(modName, instName string, _, _ []string) (module.Module, error) { + return &Chain{ + modName: modName, + instName: instName, + }, nil +} + +func (s *Chain) Init(cfg *config.Map) error { + cfg.Callback("step", func(m *config.Map, node config.Node) error { + var tbl module.Table + err := modconfig.ModuleFromNode("table", node.Args, node, m.Globals, &tbl) + if err != nil { + return err + } + + s.chain = append(s.chain, tbl) + s.optional = append(s.optional, false) + return nil + }) + cfg.Callback("optional_step", func(m *config.Map, node config.Node) error { + var tbl module.Table + err := modconfig.ModuleFromNode("table", node.Args, node, m.Globals, &tbl) + if err != nil { + return err + } + + s.chain = append(s.chain, tbl) + s.optional = append(s.optional, true) + return nil + }) + + _, err := cfg.Process() + return err +} + +func (s *Chain) Name() string { + return s.modName +} + +func (s *Chain) InstanceName() string { + return s.instName +} + +func (s *Chain) Lookup(ctx context.Context, key string) (string, bool, error) { + newVal, err := s.LookupMulti(ctx, key) + if err != nil { + return "", false, err + } + if len(newVal) == 0 { + return "", false, nil + } + + return newVal[0], true, nil +} + +func (s *Chain) LookupMulti(ctx context.Context, key string) ([]string, error) { + result := []string{key} +STEP: + for i, step := range s.chain { + newResult := []string{} + for _, key = range result { + if step_multi, ok := step.(module.MultiTable); ok { + val, err := step_multi.LookupMulti(ctx, key) + if err != nil { + return []string{}, err + } + if len(val) == 0 { + if s.optional[i] { + continue STEP + } + return []string{}, nil + } + newResult = append(newResult, val...) + } else { + val, ok, err := step.Lookup(ctx, key) + if err != nil { + return []string{}, err + } + if !ok { + if s.optional[i] { + continue STEP + } + return []string{}, nil + } + newResult = append(newResult, val) + } + } + result = newResult + } + return result, nil +} + +func init() { + module.Register("table.chain", NewChain) +} diff --git a/internal/table/email_localpart.go b/internal/table/email_localpart.go new file mode 100644 index 0000000..a9d6f06 --- /dev/null +++ b/internal/table/email_localpart.go @@ -0,0 +1,70 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +type EmailLocalpart struct { + modName string + instName string + allowNonEmail bool +} + +func NewEmailLocalpart(modName, instName string, _, _ []string) (module.Module, error) { + return &EmailLocalpart{ + modName: modName, + instName: instName, + allowNonEmail: modName == "table.email_localpart_optional", + }, nil +} + +func (s *EmailLocalpart) Init(cfg *config.Map) error { + return nil +} + +func (s *EmailLocalpart) Name() string { + return s.modName +} + +func (s *EmailLocalpart) InstanceName() string { + return s.modName +} + +func (s *EmailLocalpart) Lookup(ctx context.Context, key string) (string, bool, error) { + mbox, _, err := address.Split(key) + if err != nil { + if s.allowNonEmail { + return key, true, nil + } + // Invalid email, no local part mapping. + return "", false, nil + } + return mbox, true, nil +} + +func init() { + module.Register("table.email_localpart", NewEmailLocalpart) + module.Register("table.email_localpart_optional", NewEmailLocalpart) +} diff --git a/internal/table/email_with_domain.go b/internal/table/email_with_domain.go new file mode 100644 index 0000000..4eb50b5 --- /dev/null +++ b/internal/table/email_with_domain.go @@ -0,0 +1,89 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + "fmt" + + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +type EmailWithDomain struct { + modName string + instName string + domains []string + log log.Logger +} + +func NewEmailWithDomain(modName, instName string, _, inlineArgs []string) (module.Module, error) { + return &EmailWithDomain{ + modName: modName, + instName: instName, + domains: inlineArgs, + log: log.Logger{Name: modName}, + }, nil +} + +func (s *EmailWithDomain) Init(cfg *config.Map) error { + for _, d := range s.domains { + if !address.ValidDomain(d) { + return fmt.Errorf("%s: invalid domain: %s", s.modName, d) + } + } + if len(s.domains) == 0 { + return fmt.Errorf("%s: at least one domain is required", s.modName) + } + + return nil +} + +func (s *EmailWithDomain) Name() string { + return s.modName +} + +func (s *EmailWithDomain) InstanceName() string { + return s.modName +} + +func (s *EmailWithDomain) Lookup(ctx context.Context, key string) (string, bool, error) { + quotedMbox := address.QuoteMbox(key) + + if len(s.domains) == 0 { + s.log.Msg("only first domain is used when expanding key", "key", key, "domain", s.domains[0]) + } + + return quotedMbox + "@" + s.domains[0], true, nil +} + +func (s *EmailWithDomain) LookupMulti(ctx context.Context, key string) ([]string, error) { + quotedMbox := address.QuoteMbox(key) + emails := make([]string, len(s.domains)) + for i, domain := range s.domains { + emails[i] = quotedMbox + "@" + domain + } + return emails, nil +} + +func init() { + module.Register("table.email_with_domain", NewEmailWithDomain) +} diff --git a/internal/table/file.go b/internal/table/file.go new file mode 100644 index 0000000..a0286a9 --- /dev/null +++ b/internal/table/file.go @@ -0,0 +1,258 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "bufio" + "context" + "fmt" + "os" + "runtime/debug" + "strings" + "sync" + "time" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +const FileModName = "table.file" + +type File struct { + instName string + file string + + m map[string][]string + mLck sync.RWMutex + mStamp time.Time + + stopReloader chan struct{} + forceReload chan struct{} + + log log.Logger +} + +func NewFile(_, instName string, _, inlineArgs []string) (module.Module, error) { + m := &File{ + instName: instName, + m: make(map[string][]string), + stopReloader: make(chan struct{}), + forceReload: make(chan struct{}), + log: log.Logger{Name: FileModName}, + } + + switch len(inlineArgs) { + case 1: + m.file = inlineArgs[0] + case 0: + default: + return nil, fmt.Errorf("%s: cannot use multiple files with single %s, use %s multiple times to do so", FileModName, FileModName, FileModName) + } + + return m, nil +} + +func (f *File) Name() string { + return FileModName +} + +func (f *File) InstanceName() string { + return f.instName +} + +func (f *File) Init(cfg *config.Map) error { + var file string + cfg.Bool("debug", true, false, &f.log.Debug) + cfg.String("file", false, false, "", &file) + if _, err := cfg.Process(); err != nil { + return err + } + + if file != "" { + if f.file != "" { + return fmt.Errorf("%s: file path specified both in directive and in argument, do it once", FileModName) + } + f.file = file + } + + if err := readFile(f.file, f.m); err != nil { + if !os.IsNotExist(err) { + return err + } + f.log.Printf("ignoring non-existent file: %s", f.file) + } + + go f.reloader() + hooks.AddHook(hooks.EventReload, func() { + f.forceReload <- struct{}{} + }) + + return nil +} + +var reloadInterval = 15 * time.Second + +func (f *File) reloader() { + defer func() { + if err := recover(); err != nil { + stack := debug.Stack() + log.Printf("panic during m reload: %v\n%s", err, stack) + } + }() + + t := time.NewTicker(reloadInterval) + defer t.Stop() + + for { + select { + case <-t.C: + f.reload() + + case <-f.forceReload: + f.reload() + + case <-f.stopReloader: + f.stopReloader <- struct{}{} + return + } + } +} + +func (f *File) reload() { + info, err := os.Stat(f.file) + if err != nil { + if os.IsNotExist(err) { + f.mLck.Lock() + f.m = map[string][]string{} + f.mLck.Unlock() + return + } + f.log.Error("os stat", err) + } + if info.ModTime().Before(f.mStamp) || time.Since(info.ModTime()) < (reloadInterval/2) { + return // reload not necessary + } + + f.log.Debugf("reloading") + + newm := make(map[string][]string, len(f.m)+5) + if err := readFile(f.file, newm); err != nil { + if os.IsNotExist(err) { + f.log.Printf("ignoring non-existent file: %s", f.file) + return + } + + f.log.Println(err) + return + } + // after reading we need to check whether file has changed in between + info2, err := os.Stat(f.file) + if err != nil { + f.log.Println(err) + return + } + + if !info2.ModTime().Equal(info.ModTime()) { + // file has changed in the meantime + return + } + + f.mLck.Lock() + f.m = newm + f.mStamp = info.ModTime() + f.mLck.Unlock() +} + +func (f *File) Close() error { + f.stopReloader <- struct{}{} + <-f.stopReloader + return nil +} + +func readFile(path string, out map[string][]string) error { + f, err := os.Open(path) + if err != nil { + return err + } + + scnr := bufio.NewScanner(f) + lineCounter := 0 + + parseErr := func(text string) error { + return fmt.Errorf("%s:%d: %s", path, lineCounter, text) + } + + for scnr.Scan() { + lineCounter++ + if strings.HasPrefix(scnr.Text(), "#") { + continue + } + + text := strings.TrimSpace(scnr.Text()) + if text == "" { + continue + } + + parts := strings.SplitN(text, ":", 2) + if len(parts) == 1 { + parts = append(parts, "") + } + + from := strings.TrimSpace(parts[0]) + if len(from) == 0 { + return parseErr("empty address before colon") + } + + for _, to := range strings.Split(parts[1], ",") { + to := strings.TrimSpace(to) + out[from] = append(out[from], to) + } + } + return scnr.Err() +} + +func (f *File) Lookup(_ context.Context, val string) (string, bool, error) { + // The existing map is never modified, instead it is replaced with a new + // one if reload is performed. + f.mLck.RLock() + usedFile := f.m + f.mLck.RUnlock() + + newVal, ok := usedFile[val] + + if len(newVal) == 0 { + return "", false, nil + } + + return newVal[0], ok, nil +} + +func (f *File) LookupMulti(_ context.Context, val string) ([]string, error) { + f.mLck.RLock() + usedFile := f.m + f.mLck.RUnlock() + + return usedFile[val], nil +} + +func init() { + module.Register(FileModName, NewFile) +} diff --git a/internal/table/file_test.go b/internal/table/file_test.go new file mode 100644 index 0000000..c51620d --- /dev/null +++ b/internal/table/file_test.go @@ -0,0 +1,222 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "os" + "reflect" + "testing" + "time" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestReadFile(t *testing.T) { + test := func(file string, expected map[string][]string) { + t.Helper() + + f, err := os.CreateTemp("", "maddy-tests-") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + defer f.Close() + if _, err := f.WriteString(file); err != nil { + t.Fatal(err) + } + + actual := map[string][]string{} + err = readFile(f.Name(), actual) + if expected == nil { + if err == nil { + t.Errorf("expected failure, got %+v", actual) + } + return + } + if err != nil { + t.Errorf("unexpected failure: %v", err) + return + } + + if !reflect.DeepEqual(actual, expected) { + t.Errorf("wrong results\n want %+v\n got %+v", expected, actual) + } + } + + test("a: b", map[string][]string{"a": {"b"}}) + test("a@example.org: b@example.com", map[string][]string{"a@example.org": {"b@example.com"}}) + test(`"a @ a"@example.org: b@example.com`, map[string][]string{`"a @ a"@example.org`: {"b@example.com"}}) + test(`a@example.org: "b @ b"@example.com`, map[string][]string{`a@example.org`: {`"b @ b"@example.com`}}) + test(`"a @ a": "b @ b"`, map[string][]string{`"a @ a"`: {`"b @ b"`}}) + test("a: b, c", map[string][]string{"a": {"b", "c"}}) + test("a: b\na: c", map[string][]string{"a": {"b", "c"}}) + test(": b", nil) + test(":", nil) + test("aaa", map[string][]string{"aaa": {""}}) + test(": b", nil) + test(" testing@example.com : arbitrary-whitespace@example.org ", + map[string][]string{"testing@example.com": {"arbitrary-whitespace@example.org"}}) + test(`# skip comments +a: b`, map[string][]string{"a": {"b"}}) + test(`# and empty lines + +a: b`, map[string][]string{"a": {"b"}}) + test("# with whitespace too\n \na: b", map[string][]string{"a": {"b"}}) + test("a: b\na: c", map[string][]string{"a": {"b", "c"}}) +} + +func TestFileReload(t *testing.T) { + t.Parallel() + + const file = `cat: dog` + + f, err := os.CreateTemp("", "maddy-tests-") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + if _, err := f.WriteString(file); err != nil { + f.Close() + t.Fatal(err) + } + f.Close() + + mod, err := NewFile("", "", nil, []string{f.Name()}) + if err != nil { + t.Fatal(err) + } + m := mod.(*File) + m.log = testutils.Logger(t, "file_map") + defer m.Close() + + if err := mod.Init(&config.Map{Block: config.Node{}}); err != nil { + t.Fatal(err) + } + + // ensure it is correctly loaded at first time. + m.mLck.RLock() + if m.m["cat"] == nil { + t.Fatalf("wrong content loaded, new m were not loaded, %v", m.m) + } + m.mLck.RUnlock() + + for i := 0; i < 100; i++ { + // try to provoke race condition on file writing + if i%2 == 0 { + if err := os.WriteFile(f.Name(), []byte("dog: cat"), os.ModePerm); err != nil { + t.Fatal(err) + } + } + time.Sleep(reloadInterval + 5*time.Millisecond) + m.mLck.RLock() + if m.m["dog"] == nil { + t.Fatalf("wrong content loaded, new m were not loaded, %v", m.m) + } + m.mLck.RUnlock() + } +} + +func TestFileReload_Broken(t *testing.T) { + t.Parallel() + + const file = `cat: dog` + + f, err := os.CreateTemp("", "maddy-tests-") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + if _, err := f.WriteString(file); err != nil { + f.Close() + t.Fatal(err) + } + f.Close() + + mod, err := NewFile("", "", nil, []string{f.Name()}) + if err != nil { + t.Fatal(err) + } + m := mod.(*File) + m.log = testutils.Logger(t, FileModName) + defer m.Close() + + if err := mod.Init(&config.Map{Block: config.Node{}}); err != nil { + t.Fatal(err) + } + + f2, err := os.OpenFile(f.Name(), os.O_WRONLY|os.O_SYNC, os.ModePerm) + if err != nil { + t.Fatal(err) + } + if _, err := f2.WriteString(":"); err != nil { + t.Fatal(err) + } + defer f2.Close() + + time.Sleep(3 * reloadInterval) + + m.mLck.RLock() + defer m.mLck.RUnlock() + if m.m["cat"] == nil { + t.Fatal("New m were loaded or map changed", m.m) + } +} + +func TestFileReload_Removed(t *testing.T) { + t.Parallel() + + const file = `cat: dog` + + f, err := os.CreateTemp("", "maddy-tests-") + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteString(file); err != nil { + f.Close() + t.Fatal(err) + } + f.Close() + + mod, err := NewFile("", "", nil, []string{f.Name()}) + if err != nil { + t.Fatal(err) + } + m := mod.(*File) + m.log = testutils.Logger(t, FileModName) + defer m.Close() + + if err := mod.Init(&config.Map{Block: config.Node{}}); err != nil { + t.Fatal(err) + } + + os.Remove(f.Name()) + + time.Sleep(3 * reloadInterval) + + m.mLck.RLock() + defer m.mLck.RUnlock() + if m.m["cat"] != nil { + t.Fatal("Old m are still loaded") + } +} + +func init() { + reloadInterval = 10 * time.Millisecond +} diff --git a/internal/table/identity.go b/internal/table/identity.go new file mode 100644 index 0000000..c405d3d --- /dev/null +++ b/internal/table/identity.go @@ -0,0 +1,58 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +type Identity struct { + modName string + instName string +} + +func NewIdentity(modName, instName string, _, _ []string) (module.Module, error) { + return &Identity{ + modName: modName, + instName: instName, + }, nil +} + +func (s *Identity) Init(cfg *config.Map) error { + return nil +} + +func (s *Identity) Name() string { + return s.modName +} + +func (s *Identity) InstanceName() string { + return s.modName +} + +func (s *Identity) Lookup(_ context.Context, key string) (string, bool, error) { + return key, true, nil +} + +func init() { + module.Register("table.identity", NewIdentity) +} diff --git a/internal/table/regexp.go b/internal/table/regexp.go new file mode 100644 index 0000000..069be6c --- /dev/null +++ b/internal/table/regexp.go @@ -0,0 +1,127 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + "fmt" + "regexp" + "strings" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +type Regexp struct { + modName string + instName string + inlineArgs []string + + re *regexp.Regexp + replacements []string + + expandPlaceholders bool +} + +func NewRegexp(modName, instName string, _, inlineArgs []string) (module.Module, error) { + return &Regexp{ + modName: modName, + instName: instName, + inlineArgs: inlineArgs, + }, nil +} + +func (r *Regexp) Init(cfg *config.Map) error { + var ( + fullMatch bool + caseInsensitive bool + ) + cfg.Bool("full_match", false, true, &fullMatch) + cfg.Bool("case_insensitive", false, true, &caseInsensitive) + cfg.Bool("expand_replaceholders", false, true, &r.expandPlaceholders) + if _, err := cfg.Process(); err != nil { + return err + } + + regex := r.inlineArgs[0] + if len(r.inlineArgs) > 1 { + r.replacements = r.inlineArgs[1:] + } + + if fullMatch { + if !strings.HasPrefix(regex, "^") { + regex = "^" + regex + } + if !strings.HasSuffix(regex, "$") { + regex = regex + "$" + } + } + + if caseInsensitive { + regex = "(?i)" + regex + } + + var err error + r.re, err = regexp.Compile(regex) + if err != nil { + return fmt.Errorf("%s: %v", r.modName, err) + } + return nil +} + +func (r *Regexp) Name() string { + return r.modName +} + +func (r *Regexp) InstanceName() string { + return r.modName +} + +func (r *Regexp) LookupMulti(_ context.Context, key string) ([]string, error) { + matches := r.re.FindStringSubmatchIndex(key) + if matches == nil { + return []string{}, nil + } + + result := []string{} + for _, replacement := range r.replacements { + if !r.expandPlaceholders { + result = append(result, replacement) + } else { + result = append(result, string(r.re.ExpandString([]byte{}, replacement, key, matches))) + } + } + return result, nil +} + +func (r *Regexp) Lookup(ctx context.Context, key string) (string, bool, error) { + newVal, err := r.LookupMulti(ctx, key) + if err != nil { + return "", false, err + } + if len(newVal) == 0 { + return "", false, nil + } + + return newVal[0], true, nil +} + +func init() { + module.Register("table.regexp", NewRegexp) +} diff --git a/internal/table/sql_query.go b/internal/table/sql_query.go new file mode 100644 index 0000000..c15f710 --- /dev/null +++ b/internal/table/sql_query.go @@ -0,0 +1,251 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + _ "github.com/lib/pq" +) + +type SQL struct { + modName string + instName string + + namedArgs bool + + db *sql.DB + lookup *sql.Stmt + add *sql.Stmt + list *sql.Stmt + set *sql.Stmt + del *sql.Stmt +} + +func NewSQL(modName, instName string, _, _ []string) (module.Module, error) { + return &SQL{ + modName: modName, + instName: instName, + }, nil +} + +func (s *SQL) Name() string { + return s.modName +} + +func (s *SQL) InstanceName() string { + return s.instName +} + +func (s *SQL) Init(cfg *config.Map) error { + var ( + driver string + initQueries []string + dsnParts []string + lookupQuery string + + addQuery string + listQuery string + removeQuery string + setQuery string + ) + cfg.StringList("init", false, false, nil, &initQueries) + cfg.String("driver", false, true, "", &driver) + cfg.StringList("dsn", false, true, nil, &dsnParts) + cfg.Bool("named_args", false, false, &s.namedArgs) + + cfg.String("lookup", false, true, "", &lookupQuery) + + cfg.String("add", false, false, "", &addQuery) + cfg.String("list", false, false, "", &listQuery) + cfg.String("del", false, false, "", &removeQuery) + cfg.String("set", false, false, "", &setQuery) + if _, err := cfg.Process(); err != nil { + return err + } + + if driver == "postgres" && s.namedArgs { + return config.NodeErr(cfg.Block, "PostgreSQL driver does not support named_args") + } + + db, err := sql.Open(driver, strings.Join(dsnParts, " ")) + if err != nil { + return config.NodeErr(cfg.Block, "failed to open db: %v", err) + } + s.db = db + + for _, init := range initQueries { + if _, err := db.Exec(init); err != nil { + return config.NodeErr(cfg.Block, "init query failed: %v", err) + } + } + + s.lookup, err = db.Prepare(lookupQuery) + if err != nil { + return config.NodeErr(cfg.Block, "failed to prepare lookup query: %v", err) + } + if addQuery != "" { + s.add, err = db.Prepare(addQuery) + if err != nil { + return config.NodeErr(cfg.Block, "failed to prepare add query: %v", err) + } + } + if listQuery != "" { + s.list, err = db.Prepare(listQuery) + if err != nil { + return config.NodeErr(cfg.Block, "failed to prepare list query: %v", err) + } + } + if setQuery != "" { + s.set, err = db.Prepare(setQuery) + if err != nil { + return config.NodeErr(cfg.Block, "failed to prepare set query: %v", err) + } + } + if removeQuery != "" { + s.del, err = db.Prepare(removeQuery) + if err != nil { + return config.NodeErr(cfg.Block, "failed to prepare del query: %v", err) + } + } + + return nil +} + +func (s *SQL) Close() error { + s.lookup.Close() + return s.db.Close() +} + +func (s *SQL) Lookup(ctx context.Context, val string) (string, bool, error) { + var ( + repl string + row *sql.Row + ) + if s.namedArgs { + row = s.lookup.QueryRowContext(ctx, sql.Named("key", val)) + } else { + row = s.lookup.QueryRowContext(ctx, val) + } + if err := row.Scan(&repl); err != nil { + if err == sql.ErrNoRows { + return "", false, nil + } + return "", false, fmt.Errorf("%s: lookup %s: %w", s.modName, val, err) + } + return repl, true, nil +} + +func (s *SQL) LookupMulti(ctx context.Context, val string) ([]string, error) { + var ( + repl []string + rows *sql.Rows + err error + ) + if s.namedArgs { + rows, err = s.lookup.QueryContext(ctx, sql.Named("key", val)) + } else { + rows, err = s.lookup.QueryContext(ctx, val) + } + if err != nil { + return nil, fmt.Errorf("%s; lookup %s: %w", s.modName, val, err) + } + for rows.Next() { + var res string + if err := rows.Scan(&res); err != nil { + return nil, fmt.Errorf("%s; lookup %s: %w", s.modName, val, err) + } + repl = append(repl, res) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("%s; lookup %s: %w", s.modName, val, err) + } + return repl, nil +} + +func (s *SQL) Keys() ([]string, error) { + if s.list == nil { + return nil, fmt.Errorf("%s: table is not mutable (no 'list' query)", s.modName) + } + + rows, err := s.list.Query() + if err != nil { + return nil, fmt.Errorf("%s: list: %w", s.modName, err) + } + defer rows.Close() + var list []string + for rows.Next() { + var key string + if err := rows.Scan(&key); err != nil { + return nil, fmt.Errorf("%s: list: %w", s.modName, err) + } + list = append(list, key) + } + return list, nil +} + +func (s *SQL) RemoveKey(k string) error { + if s.del == nil { + return fmt.Errorf("%s: table is not mutable (no 'del' query)", s.modName) + } + + var err error + if s.namedArgs { + _, err = s.del.Exec(sql.Named("key", k)) + } else { + _, err = s.del.Exec(k) + } + if err != nil { + return fmt.Errorf("%s: del %s: %w", s.modName, k, err) + } + return nil +} + +func (s *SQL) SetKey(k, v string) error { + if s.set == nil { + return fmt.Errorf("%s: table is not mutable (no 'set' query)", s.modName) + } + if s.add == nil { + return fmt.Errorf("%s: table is not mutable (no 'add' query)", s.modName) + } + + var args []interface{} + if s.namedArgs { + args = []interface{}{sql.Named("key", k), sql.Named("value", v)} + } else { + args = []interface{}{k, v} + } + + if _, err := s.add.Exec(args...); err != nil { + if _, err := s.set.Exec(args...); err != nil { + return fmt.Errorf("%s: add %s: %w", s.modName, k, err) + } + return nil + } + return nil +} + +func init() { + module.Register("table.sql_query", NewSQL) +} diff --git a/internal/table/sql_query_test.go b/internal/table/sql_query_test.go new file mode 100644 index 0000000..fd160f7 --- /dev/null +++ b/internal/table/sql_query_test.go @@ -0,0 +1,96 @@ +//go:build !nosqlite3 && cgo +// +build !nosqlite3,cgo + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + "path/filepath" + "reflect" + "testing" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestSQL(t *testing.T) { + path := testutils.Dir(t) + mod, err := NewSQL("sql_table", "", nil, nil) + if err != nil { + t.Fatal("Module create failed:", err) + } + tbl := mod.(*SQL) + err = tbl.Init(config.NewMap(nil, config.Node{ + Children: []config.Node{ + { + Name: "driver", + Args: []string{"sqlite3"}, + }, + { + Name: "dsn", + Args: []string{filepath.Join(path, "test.db")}, + }, + { + Name: "init", + Args: []string{ + "CREATE TABLE testTbl (key TEXT, value TEXT)", + "INSERT INTO testTbl VALUES ('user1', 'user1a')", + "INSERT INTO testTbl VALUES ('user1', 'user1b')", + "INSERT INTO testTbl VALUES ('user3', NULL)", + }, + }, + { + Name: "lookup", + Args: []string{"SELECT value FROM testTbl WHERE key = $key"}, + }, + }, + })) + if err != nil { + t.Fatal("Init failed:", err) + } + + check := func(key, res string, ok, fail bool) { + t.Helper() + + actualRes, actualOk, err := tbl.Lookup(context.Background(), key) + if actualRes != res { + t.Errorf("Result mismatch: want %s, got %s", res, actualRes) + } + if actualOk != ok { + t.Errorf("OK mismatch: want %v, got %v", actualOk, ok) + } + if (err != nil) != fail { + t.Errorf("Error mismatch: want failure = %v, got %v", fail, err) + } + } + + check("user1", "user1a", true, false) + check("user2", "", false, false) + check("user3", "", false, true) + + vals, err := tbl.LookupMulti(context.Background(), "user1") + if err != nil { + t.Error("Unexpected error:", err) + } + if !reflect.DeepEqual(vals, []string{"user1a", "user1b"}) { + t.Error("Wrong result of LookupMulti:", vals) + } +} diff --git a/internal/table/sql_table.go b/internal/table/sql_table.go new file mode 100644 index 0000000..e793dc4 --- /dev/null +++ b/internal/table/sql_table.go @@ -0,0 +1,173 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + "fmt" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + _ "github.com/lib/pq" +) + +type SQLTable struct { + modName string + instName string + + wrapped *SQL +} + +func NewSQLTable(modName, instName string, _, _ []string) (module.Module, error) { + return &SQLTable{ + modName: modName, + instName: instName, + + wrapped: &SQL{ + modName: modName, + instName: instName, + }, + }, nil +} + +func (s *SQLTable) Name() string { + return s.modName +} + +func (s *SQLTable) InstanceName() string { + return s.instName +} + +func (s *SQLTable) Init(cfg *config.Map) error { + var ( + driver string + dsnParts []string + tableName string + keyColumn string + valueColumn string + ) + cfg.String("driver", false, true, "", &driver) + cfg.StringList("dsn", false, true, nil, &dsnParts) + cfg.String("table_name", false, true, "", &tableName) + cfg.String("key_column", false, false, "key", &keyColumn) + cfg.String("value_column", false, false, "value", &valueColumn) + if _, err := cfg.Process(); err != nil { + return err + } + + // sql_table module literally wraps the sql_query module by generating a + // configuration block for it. + + var ( + useNamedArgs string + + lookupQuery string + addQuery string + listQuery string + setQuery string + delQuery string + ) + if driver == "sqlite3" { + useNamedArgs = "yes" + lookupQuery = fmt.Sprintf("SELECT %s FROM %s WHERE %s = :key", valueColumn, tableName, keyColumn) + addQuery = fmt.Sprintf("INSERT INTO %s(%s, %s) VALUES(:key, :value)", tableName, keyColumn, valueColumn) + listQuery = fmt.Sprintf("SELECT %s from %s", keyColumn, tableName) + setQuery = fmt.Sprintf("UPDATE %s SET %s = :value WHERE %s = :key", tableName, valueColumn, keyColumn) + delQuery = fmt.Sprintf("DELETE FROM %s WHERE %s = :key", tableName, keyColumn) + } else { + useNamedArgs = "no" + lookupQuery = fmt.Sprintf("SELECT %s FROM %s WHERE %s = $1", valueColumn, tableName, keyColumn) + addQuery = fmt.Sprintf("INSERT INTO %s(%s, %s) VALUES($1, $2)", tableName, keyColumn, valueColumn) + listQuery = fmt.Sprintf("SELECT %s from %s", keyColumn, tableName) + setQuery = fmt.Sprintf("UPDATE %s SET %s = $2 WHERE %s = $1", tableName, valueColumn, keyColumn) + delQuery = fmt.Sprintf("DELETE FROM %s WHERE %s = $1", tableName, keyColumn) + } + + return s.wrapped.Init(config.NewMap(cfg.Globals, config.Node{ + Children: []config.Node{ + { + Name: "driver", + Args: []string{driver}, + }, + { + Name: "dsn", + Args: dsnParts, + }, + { + Name: "named_args", + Args: []string{useNamedArgs}, + }, + { + Name: "lookup", + Args: []string{lookupQuery}, + }, + { + Name: "add", + Args: []string{addQuery}, + }, + { + Name: "list", + Args: []string{listQuery}, + }, + { + Name: "set", + Args: []string{setQuery}, + }, + { + Name: "del", + Args: []string{delQuery}, + }, + { + Name: "init", + Args: []string{fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( + %s TEXT PRIMARY KEY NOT NULL, + %s TEXT NOT NULL + )`, tableName, keyColumn, valueColumn)}, + }, + }, + })) +} + +func (s *SQLTable) Close() error { + return s.wrapped.Close() +} + +func (s *SQLTable) Lookup(ctx context.Context, val string) (string, bool, error) { + return s.wrapped.Lookup(ctx, val) +} + +func (s *SQLTable) LookupMulti(ctx context.Context, val string) ([]string, error) { + return s.wrapped.LookupMulti(ctx, val) +} + +func (s *SQLTable) Keys() ([]string, error) { + return s.wrapped.Keys() +} + +func (s *SQLTable) RemoveKey(k string) error { + return s.wrapped.RemoveKey(k) +} + +func (s *SQLTable) SetKey(k, v string) error { + return s.wrapped.SetKey(k, v) +} + +func init() { + module.Register("table.sql_table", NewSQLTable) +} diff --git a/internal/table/sqlite3.go b/internal/table/sqlite3.go new file mode 100644 index 0000000..8b22794 --- /dev/null +++ b/internal/table/sqlite3.go @@ -0,0 +1,24 @@ +//go:build !nosqlite3 && cgo +// +build !nosqlite3,cgo + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import _ "github.com/mattn/go-sqlite3" diff --git a/internal/table/static.go b/internal/table/static.go new file mode 100644 index 0000000..e55ffd5 --- /dev/null +++ b/internal/table/static.go @@ -0,0 +1,77 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +type Static struct { + modName string + instName string + + m map[string][]string +} + +func NewStatic(modName, instName string, _, _ []string) (module.Module, error) { + return &Static{ + modName: modName, + instName: instName, + m: map[string][]string{}, + }, nil +} + +func (s *Static) Init(cfg *config.Map) error { + cfg.Callback("entry", func(_ *config.Map, node config.Node) error { + if len(node.Args) < 2 { + return config.NodeErr(node, "expected at least one value") + } + s.m[node.Args[0]] = node.Args[1:] + return nil + }) + _, err := cfg.Process() + return err +} + +func (s *Static) Name() string { + return s.modName +} + +func (s *Static) InstanceName() string { + return s.modName +} + +func (s *Static) Lookup(ctx context.Context, key string) (string, bool, error) { + val := s.m[key] + if len(val) == 0 { + return "", false, nil + } + return val[0], true, nil +} + +func (s *Static) LookupMulti(ctx context.Context, key string) ([]string, error) { + return s.m[key], nil +} + +func init() { + module.Register("table.static", NewStatic) +} diff --git a/internal/target/delivery.go b/internal/target/delivery.go new file mode 100644 index 0000000..1c3450f --- /dev/null +++ b/internal/target/delivery.go @@ -0,0 +1,34 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package target + +import ( + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +func DeliveryLogger(l log.Logger, msgMeta *module.MsgMetadata) log.Logger { + fields := make(map[string]interface{}, len(l.Fields)+1) + for k, v := range l.Fields { + fields[k] = v + } + fields["msg_id"] = msgMeta.ID + l.Fields = fields + return l +} diff --git a/internal/target/queue/metrics.go b/internal/target/queue/metrics.go new file mode 100644 index 0000000..32ada20 --- /dev/null +++ b/internal/target/queue/metrics.go @@ -0,0 +1,35 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package queue + +import "github.com/prometheus/client_golang/prometheus" + +var queuedMsgs = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "maddy", + Subsystem: "queue", + Name: "length", + Help: "Amount of queued messages", + }, + []string{"module", "location"}, +) + +func init() { + prometheus.MustRegister(queuedMsgs) +} diff --git a/internal/target/queue/queue.go b/internal/target/queue/queue.go new file mode 100644 index 0000000..264a70f --- /dev/null +++ b/internal/target/queue/queue.go @@ -0,0 +1,998 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +/* +Package queue implements module which keeps messages on disk and tries delivery +to the configured target (usually remote) multiple times until all recipients +are succeeded. + +Interfaces implemented: +- module.DeliveryTarget + +Implementation summary follows. + +All scheduled deliveries are attempted to the configured DeliveryTarget. +All metadata is preserved on disk. + +Failure status is determined on per-recipient basis: + - Delivery.Start fail handled as a failure for all recipients. + - Delivery.AddRcpt fail handled as a failure for the corresponding recipient. + - Delivery.Body fail handled as a failure for all recipients. + - If Delivery implements PartialDelivery, then + PartialDelivery.BodyNonAtomic is used instead. Failures are determined based + on StatusCollector.SetStatus calls done by target in this case. + +For each failure check is done to see if it is a permanent failure +or a temporary one. This is done using exterrors.IsTemporaryOrUnspec. +That is, errors are assumed to be temporary by default. +All errors are converted to SMTPError then due to a storage limitations. + +If there are any *temporary* failed recipients, delivery will be retried +after delay *only for these* recipients. + +Last error for each recipient is saved for reporting in NDN. A NDN is generated +if there are any failed recipients left after +last attempt to deliver the message. + +Amount of attempts for each message is limited to a certain configured number. +After last attempt, all recipients that are still temporary failing are assumed +to be permanently failed. +*/ +package queue + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "runtime" + "runtime/debug" + "runtime/trace" + "strconv" + "strings" + "sync" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/dsn" + "github.com/foxcpp/maddy/internal/msgpipeline" + "github.com/foxcpp/maddy/internal/target" +) + +// partialError describes state of partially successful message delivery. +type partialError struct { + + // Underlying error objects for each recipient. + Errs map[string]error + + // Fields can be accessed without holding this lock, but only after + // target.BodyNonAtomic/Body returns. + statusLock *sync.Mutex +} + +// SetStatus implements module.StatusCollector so partialError can be +// passed directly to PartialDelivery.BodyNonAtomic. +func (pe *partialError) SetStatus(rcptTo string, err error) { + log.Debugf("PartialError.SetStatus(%s, %v)", rcptTo, err) + if err == nil { + return + } + pe.statusLock.Lock() + defer pe.statusLock.Unlock() + pe.Errs[rcptTo] = err +} + +func (pe partialError) Error() string { + return fmt.Sprintf("delivery failed for some recipients: %v", pe.Errs) +} + +// dontRecover controls the behavior of panic handlers, if it is set to true - +// they are disabled and so tests will panic to avoid masking bugs. +var dontRecover = false + +type Queue struct { + name string + location string + hostname string + autogenMsgDomain string + wheel *TimeWheel + + dsnPipeline module.DeliveryTarget + + // Retry delay is calculated using the following formula: + // initialRetryTime * retryTimeScale ^ (TriesCount - 1) + + initialRetryTime time.Duration + retryTimeScale float64 + maxTries int + + // If any delivery is scheduled in less than postInitDelay + // after Init, its delay will be increased by postInitDelay. + // + // Say, if postInitDelay is 10 secs. + // Then if some message is scheduled to delivered 5 seconds + // after init, it will be actually delivered 15 seconds + // after start-up. + // + // This delay is added to make that if maddy is killed shortly + // after start-up for whatever reason it will not affect the queue. + postInitDelay time.Duration + + Log log.Logger + Target module.DeliveryTarget + + deliveryWg sync.WaitGroup + // Buffered channel used to restrict count of deliveries attempted + // in parallel. + deliverySemaphore chan struct{} +} + +type QueueMetadata struct { + MsgMeta *module.MsgMetadata + From string + + // Recipients that should be tried next. + // May or may not be equal to partialError.TemporaryFailed. + To []string + + // Information about previous failures. + // Preserved to be included in a bounce message. + FailedRcpts []string + TemporaryFailedRcpts []string + // All errors are converted to SMTPError we can serialize and + // also it is directly usable for bounce messages. + RcptErrs map[string]*smtp.SMTPError + + // Amount of times delivery *already tried*. + TriesCount map[string]int + + FirstAttempt time.Time + LastAttempt time.Time +} + +type queueSlot struct { + ID string + + // If nil - Hdr and Body are invalid, all values should be read from + // disk. + Meta *QueueMetadata + Hdr *textproto.Header + Body buffer.Buffer +} + +func NewQueue(_, instName string, _, inlineArgs []string) (module.Module, error) { + q := &Queue{ + name: instName, + initialRetryTime: 15 * time.Minute, + retryTimeScale: 1.25, + postInitDelay: 10 * time.Second, + Log: log.Logger{Name: "queue"}, + } + switch len(inlineArgs) { + case 0: + // Not inline definition. + case 1: + q.location = inlineArgs[0] + default: + return nil, errors.New("queue: wrong amount of inline arguments") + } + return q, nil +} + +func (q *Queue) Init(cfg *config.Map) error { + var maxParallelism int + cfg.Bool("debug", true, false, &q.Log.Debug) + cfg.Int("max_tries", false, false, 20, &q.maxTries) + cfg.Int("max_parallelism", false, false, 16, &maxParallelism) + cfg.String("location", false, false, q.location, &q.location) + cfg.Custom("target", false, true, nil, modconfig.DeliveryDirective, &q.Target) + cfg.String("hostname", true, true, "", &q.hostname) + cfg.String("autogenerated_msg_domain", true, false, "", &q.autogenMsgDomain) + cfg.Custom("bounce", false, false, nil, func(m *config.Map, node config.Node) (interface{}, error) { + return msgpipeline.New(m.Globals, node.Children) + }, &q.dsnPipeline) + if _, err := cfg.Process(); err != nil { + return err + } + + if q.dsnPipeline != nil { + if q.autogenMsgDomain == "" { + return errors.New("queue: autogenerated_msg_domain is required if bounce {} is specified") + } + + q.dsnPipeline.(*msgpipeline.MsgPipeline).Hostname = q.hostname + q.dsnPipeline.(*msgpipeline.MsgPipeline).Log = log.Logger{Name: "queue/pipeline", Debug: q.Log.Debug} + } + if q.location == "" && q.name == "" { + return errors.New("queue: need explicit location directive or inline argument if defined inline") + } + if q.location == "" { + q.location = filepath.Join(config.StateDirectory, q.name) + } + + // TODO: Check location write permissions. + if err := os.MkdirAll(q.location, os.ModePerm); err != nil { + return err + } + + return q.start(maxParallelism) +} + +func (q *Queue) start(maxParallelism int) error { + q.wheel = NewTimeWheel(q.dispatch) + q.deliverySemaphore = make(chan struct{}, maxParallelism) + + if err := q.readDiskQueue(); err != nil { + return err + } + + q.Log.Debugf("delivery target: %T", q.Target) + + return nil +} + +func (q *Queue) Close() error { + q.wheel.Close() + q.deliveryWg.Wait() + + return nil +} + +// discardBroken changes the name of metadata file to have .meta_broken +// extension. +// +// Further attempts to deliver (due to a timewheel) it will fail due to +// non-existent meta-data file. +// +// No error handling is done since this function is called from panic handler. +func (q *Queue) discardBroken(id string) { + err := os.Rename(filepath.Join(q.location, id+".meta"), filepath.Join(q.location, id+".meta_broken")) + if err != nil { + // Note: Global logger is used in case there is something wrong with Queue.Log. + log.Printf("can't mark the queue message as broken: %v", err) + } +} + +func (q *Queue) dispatch(value TimeSlot) { + slot := value.Value.(queueSlot) + + q.Log.Debugln("starting delivery for", slot.ID) + + q.deliveryWg.Add(1) + go func() { + q.Log.Debugln("waiting on delivery semaphore for", slot.ID) + q.deliverySemaphore <- struct{}{} + defer func() { + <-q.deliverySemaphore + q.deliveryWg.Done() + + if dontRecover { + return + } + + if err := recover(); err != nil { + stack := debug.Stack() + log.Printf("panic during queue dispatch %s: %v\n%s", slot.ID, err, stack) + q.discardBroken(slot.ID) + } + }() + + q.Log.Debugln("delivery semaphore acquired for", slot.ID) + var ( + meta *QueueMetadata + hdr textproto.Header + body buffer.Buffer + ) + if slot.Meta == nil { + var err error + meta, hdr, body, err = q.openMessage(slot.ID) + if err != nil { + q.Log.Error("read message", err, slot.ID) + return + } + if meta == nil { + panic("wtf") + } + } else { + meta = slot.Meta + hdr = *slot.Hdr + body = slot.Body + } + + q.tryDelivery(meta, hdr, body) + }() +} + +func toSMTPErr(err error) *smtp.SMTPError { + if err == nil { + return nil + } + + res := &smtp.SMTPError{ + Code: 554, + EnhancedCode: smtp.EnhancedCode{5, 0, 0}, + Message: "Internal server error", + } + + if exterrors.IsTemporaryOrUnspec(err) { + res.Code = 451 + res.EnhancedCode = smtp.EnhancedCode{4, 0, 0} + } + + ctxInfo := exterrors.Fields(err) + ctxCode, ok := ctxInfo["smtp_code"].(int) + if ok { + res.Code = ctxCode + } + ctxEnchCode, ok := ctxInfo["smtp_enchcode"].(smtp.EnhancedCode) + if ok { + res.EnhancedCode = ctxEnchCode + } + ctxMsg, ok := ctxInfo["smtp_msg"].(string) + if ok { + res.Message = ctxMsg + } + + if smtpErr, ok := err.(*smtp.SMTPError); ok { + log.Printf("plain SMTP error returned, this is deprecated") + res.Code = smtpErr.Code + res.EnhancedCode = smtpErr.EnhancedCode + res.Message = smtpErr.Message + } + + return res +} + +func (q *Queue) tryDelivery(meta *QueueMetadata, header textproto.Header, body buffer.Buffer) { + dl := target.DeliveryLogger(q.Log, meta.MsgMeta) + + partialErr := q.deliver(meta, header, body) + dl.Debugf("errors: %v", partialErr.Errs) + + // While iterating the list of recipients we also pick the smallest tries count + // and use it to calculate the delay for the next attempt. + smallestTriesCount := 999999 + + if meta.TriesCount == nil { + meta.TriesCount = make(map[string]int) + } + + // Check attempted recipients and corresponding errors. + // Split list into two parts: recipients that should be retried (newRcpts) + // and recipients DSN will be generated for. + newRcpts := make([]string, 0, len(partialErr.Errs)) + failedRcpts := make([]string, 0, len(partialErr.Errs)) + for _, rcpt := range meta.To { + rcptErr, ok := partialErr.Errs[rcpt] + if !ok { + dl.Msg("delivered", "rcpt", rcpt, "attempt", meta.TriesCount[rcpt]+1) + continue + } + + // Save last error (either temporary or permanent) for reporting in the DSN. + dl.Error("delivery attempt failed", rcptErr, "rcpt", rcpt) + meta.RcptErrs[rcpt] = toSMTPErr(rcptErr) + + temporary := exterrors.IsTemporaryOrUnspec(rcptErr) + if !temporary || meta.TriesCount[rcpt]+1 >= q.maxTries { + delete(meta.TriesCount, rcpt) + dl.Msg("not delivered, permanent error", "rcpt", rcpt) + failedRcpts = append(failedRcpts, rcpt) + continue + } + + // Temporary error, increase tries counter and requeue. + meta.TriesCount[rcpt]++ + newRcpts = append(newRcpts, rcpt) + + // See smallestTriesCount comment. + if count := meta.TriesCount[rcpt]; count < smallestTriesCount { + smallestTriesCount = count + } + } + + // Generate DSN for recipients that failed permanently this time. + if len(failedRcpts) != 0 { + q.emitDSN(meta, header, failedRcpts) + } + // No recipients to try, either all failed or all succeeded. + if len(newRcpts) == 0 { + q.removeFromDisk(meta.MsgMeta) + return + } + + meta.To = newRcpts + meta.LastAttempt = time.Now() + + if err := q.updateMetadataOnDisk(meta); err != nil { + dl.Error("meta-data update", err) + } + + nextTryTime := time.Now() + // Delay between retries grows exponentally, the formula is: + // initialRetryTime * retryTimeScale ^ (smallestTriesCount - 1) + dl.Debugf("delay: %v * %v ^ (%v - 1)", q.initialRetryTime, q.retryTimeScale, smallestTriesCount) + scaleFactor := time.Duration(math.Pow(q.retryTimeScale, float64(smallestTriesCount-1))) + nextTryTime = nextTryTime.Add(q.initialRetryTime * scaleFactor) + dl.Msg("will retry", + "attempts_count", meta.TriesCount, + "next_try_delay", time.Until(nextTryTime), + "rcpts", meta.To) + + q.wheel.Add(nextTryTime, queueSlot{ + ID: meta.MsgMeta.ID, + + // Do not keep (meta-)data in memory to reduce usage. At this point, + // it is safe on disk and next try will reread it. + Meta: nil, + Hdr: nil, + Body: nil, + }) +} + +func (q *Queue) deliver(meta *QueueMetadata, header textproto.Header, body buffer.Buffer) partialError { + dl := target.DeliveryLogger(q.Log, meta.MsgMeta) + perr := partialError{ + Errs: map[string]error{}, + statusLock: new(sync.Mutex), + } + + msgMeta := meta.MsgMeta.DeepCopy() + msgMeta.ID = msgMeta.ID + "-" + strconv.FormatInt(time.Now().Unix(), 16) + dl.Debugf("using message ID = %s", msgMeta.ID) + + msgCtx, msgTask := trace.NewTask(context.Background(), "Queue delivery") + defer msgTask.End() + + mailCtx, mailTask := trace.NewTask(msgCtx, "MAIL FROM") + delivery, err := q.Target.Start(mailCtx, msgMeta, meta.From) + mailTask.End() + if err != nil { + dl.Debugf("target.Start failed: %v", err) + for _, rcpt := range meta.To { + perr.Errs[rcpt] = err + } + return perr + } + dl.Debugf("target.Start OK") + + var acceptedRcpts []string + for _, rcpt := range meta.To { + rcptCtx, rcptTask := trace.NewTask(msgCtx, "RCPT TO") + if err := delivery.AddRcpt(rcptCtx, rcpt, smtp.RcptOptions{} /* TODO: DSN support */); err != nil { + dl.Debugf("delivery.AddRcpt %s failed: %v", rcpt, err) + perr.Errs[rcpt] = err + } else { + dl.Debugf("delivery.AddRcpt %s OK", rcpt) + acceptedRcpts = append(acceptedRcpts, rcpt) + } + rcptTask.End() + } + + if len(acceptedRcpts) == 0 { + dl.Debugf("delivery.Abort (no accepted recipients)") + if err := delivery.Abort(msgCtx); err != nil { + dl.Error("delivery.Abort failed", err) + } + return perr + } + + expandToPartialErr := func(err error) { + for _, rcpt := range acceptedRcpts { + perr.Errs[rcpt] = err + } + } + + bodyCtx, bodyTask := trace.NewTask(msgCtx, "DATA") + defer bodyTask.End() + + partDelivery, ok := delivery.(module.PartialDelivery) + if ok { + dl.Debugf("using delivery.BodyNonAtomic") + partDelivery.BodyNonAtomic(bodyCtx, &perr, header, body) + } else { + if err := delivery.Body(bodyCtx, header, body); err != nil { + dl.Debugf("delivery.Body failed: %v", err) + expandToPartialErr(err) + } + dl.Debugf("delivery.Body OK") + } + + allFailed := true + for _, rcpt := range acceptedRcpts { + if perr.Errs[rcpt] == nil { + allFailed = false + } + } + if allFailed { + // No recipients succeeded. + dl.Debugf("delivery.Abort (all recipients failed)") + if err := delivery.Abort(bodyCtx); err != nil { + dl.Msg("delivery.Abort failed", err) + } + return perr + } + + if err := delivery.Commit(bodyCtx); err != nil { + dl.Debugf("delivery.Commit failed: %v", err) + expandToPartialErr(err) + } + dl.Debugf("delivery.Commit OK") + + return perr +} + +type queueDelivery struct { + q *Queue + meta *QueueMetadata + + header textproto.Header + body buffer.Buffer +} + +func (qd *queueDelivery) AddRcpt(ctx context.Context, rcptTo string, _ smtp.RcptOptions) error { + qd.meta.To = append(qd.meta.To, rcptTo) + return nil +} + +func (qd *queueDelivery) Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error { + defer trace.StartRegion(ctx, "queue/Body").End() + + // Body buffer initially passed to us may not be valid after "delivery" to queue completes. + // storeNewMessage returns a new buffer object created from message blob stored on disk. + storedBody, err := qd.q.storeNewMessage(qd.meta, header, body) + if err != nil { + return err + } + + qd.body = storedBody + qd.header = header + return nil +} + +func (qd *queueDelivery) Abort(ctx context.Context) error { + defer trace.StartRegion(ctx, "queue/Abort").End() + + if qd.body != nil { + qd.q.removeFromDisk(qd.meta.MsgMeta) + } + return nil +} + +func (qd *queueDelivery) Commit(ctx context.Context) error { + defer trace.StartRegion(ctx, "queue/Commit").End() + + if qd.meta == nil { + panic("queue: double Commit") + } + + qd.q.wheel.Add(time.Time{}, queueSlot{ + ID: qd.meta.MsgMeta.ID, + Meta: qd.meta, + Hdr: &qd.header, + Body: qd.body, + }) + qd.meta = nil + qd.body = nil + return nil +} + +func (q *Queue) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + meta := &QueueMetadata{ + MsgMeta: msgMeta, + From: mailFrom, + RcptErrs: map[string]*smtp.SMTPError{}, + FirstAttempt: time.Now(), + LastAttempt: time.Now(), + } + return &queueDelivery{q: q, meta: meta}, nil +} + +func (q *Queue) removeFromDisk(msgMeta *module.MsgMetadata) { + id := msgMeta.ID + dl := target.DeliveryLogger(q.Log, msgMeta) + + // Order is important. + // If we remove header and body but can't remove meta now - readDiskQueue + // will detect and report it. + headerPath := filepath.Join(q.location, id+".header") + if err := os.Remove(headerPath); err != nil { + dl.Error("failed to remove header from disk", err) + } + bodyPath := filepath.Join(q.location, id+".body") + if err := os.Remove(bodyPath); err != nil { + dl.Error("failed to remove body from disk", err) + } + metaPath := filepath.Join(q.location, id+".meta") + if err := os.Remove(metaPath); err != nil { + dl.Error("failed to remove meta-data from disk", err) + } + dl.Debugf("removed message from disk") +} + +func (q *Queue) readDiskQueue() error { + dirInfo, err := os.ReadDir(q.location) + if err != nil { + return err + } + + // TODO(GH #209): Rewrite this function to pass all sub-tests in TestQueueDelivery_DeserializationCleanUp/NoMeta. + + loadedCount := 0 + for _, entry := range dirInfo { + // We start loading from meta-data files and then check whether ID.header and ID.body exist. + // This allows us to properly detect dangling body files. + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".meta") { + continue + } + id := entry.Name()[:len(entry.Name())-5] + + meta, err := q.readMessageMeta(id) + if err != nil { + q.Log.Printf("failed to read meta-data, skipping: %v (msg ID = %s)", err, id) + continue + } + + // Check header file existence. + if _, err := os.Stat(filepath.Join(q.location, id+".header")); err != nil { + if os.IsNotExist(err) { + q.Log.Printf("header file doesn't exist for msg ID = %s", id) + q.tryRemoveDanglingFile(id + ".meta") + q.tryRemoveDanglingFile(id + ".body") + } else { + q.Log.Printf("skipping nonstat'able header file: %v (msg ID = %s)", err, id) + } + continue + } + + // Check body file existence. + if _, err := os.Stat(filepath.Join(q.location, id+".body")); err != nil { + if os.IsNotExist(err) { + q.Log.Printf("body file doesn't exist for msg ID = %s", id) + q.tryRemoveDanglingFile(id + ".meta") + q.tryRemoveDanglingFile(id + ".header") + } else { + q.Log.Printf("skipping nonstat'able body file: %v (msg ID = %s)", err, id) + } + continue + } + + smallestTriesCount := 999999 + for _, count := range meta.TriesCount { + if smallestTriesCount > count { + smallestTriesCount = count + } + } + nextTryTime := meta.LastAttempt + scaleFactor := time.Duration(math.Pow(q.retryTimeScale, float64(smallestTriesCount-1))) + nextTryTime = nextTryTime.Add(q.initialRetryTime * scaleFactor) + + if time.Until(nextTryTime) < q.postInitDelay { + nextTryTime = time.Now().Add(q.postInitDelay) + } + + q.Log.Debugf("will try to deliver (msg ID = %s) in %v (%v)", id, time.Until(nextTryTime), nextTryTime) + q.wheel.Add(nextTryTime, queueSlot{ + ID: id, + }) + loadedCount++ + } + + if loadedCount != 0 { + q.Log.Printf("loaded %d saved queue entries", loadedCount) + } + + return nil +} + +func (q *Queue) storeNewMessage(meta *QueueMetadata, header textproto.Header, body buffer.Buffer) (buffer.Buffer, error) { + id := meta.MsgMeta.ID + + headerPath := filepath.Join(q.location, id+".header") + headerFile, err := os.Create(headerPath) + if err != nil { + return nil, err + } + defer headerFile.Close() + + if err := textproto.WriteHeader(headerFile, header); err != nil { + q.tryRemoveDanglingFile(id + ".header") + return nil, err + } + + bodyReader, err := body.Open() + if err != nil { + q.tryRemoveDanglingFile(id + ".header") + return nil, err + } + defer bodyReader.Close() + + bodyPath := filepath.Join(q.location, id+".body") + bodyFile, err := os.Create(bodyPath) + if err != nil { + return nil, err + } + defer bodyFile.Close() + + if _, err := io.Copy(bodyFile, bodyReader); err != nil { + q.tryRemoveDanglingFile(id + ".body") + q.tryRemoveDanglingFile(id + ".header") + return nil, err + } + + if err := q.updateMetadataOnDisk(meta); err != nil { + q.tryRemoveDanglingFile(id + ".body") + q.tryRemoveDanglingFile(id + ".header") + return nil, err + } + + if err := headerFile.Sync(); err != nil { + return nil, err + } + + if err := bodyFile.Sync(); err != nil { + return nil, err + } + + return buffer.FileBuffer{Path: bodyPath, LenHint: body.Len()}, nil +} + +func (q *Queue) updateMetadataOnDisk(meta *QueueMetadata) error { + metaPath := filepath.Join(q.location, meta.MsgMeta.ID+".meta") + + var file *os.File + var err error + if runtime.GOOS == "windows" { + file, err = os.Create(metaPath) + if err != nil { + return err + } + } else { + file, err = os.Create(metaPath + ".new") + if err != nil { + return err + } + } + defer file.Close() + + metaCopy := *meta + metaCopy.MsgMeta = meta.MsgMeta.DeepCopy() + metaCopy.MsgMeta.Conn = nil + + if err := json.NewEncoder(file).Encode(metaCopy); err != nil { + return err + } + + if err := file.Sync(); err != nil { + return err + } + + if runtime.GOOS != "windows" { + if err := os.Rename(metaPath+".new", metaPath); err != nil { + return err + } + } + + return nil +} + +func (q *Queue) readMessageMeta(id string) (*QueueMetadata, error) { + metaPath := filepath.Join(q.location, id+".meta") + file, err := os.Open(metaPath) + if err != nil { + return nil, err + } + defer file.Close() + + meta := &QueueMetadata{} + + meta.MsgMeta = &module.MsgMetadata{} + + // There is a couple of problems we have to solve before we would be able to + // serialize ConnState. + // 1. future.Future can't be serialized. + // 2. net.Addr can't be deserialized because we don't know the concrete type. + + if err := json.NewDecoder(file).Decode(meta); err != nil { + return nil, err + } + + return meta, nil +} + +type BufferedReadCloser struct { + *bufio.Reader + io.Closer +} + +func (q *Queue) tryRemoveDanglingFile(name string) { + if err := os.Remove(filepath.Join(q.location, name)); err != nil { + q.Log.Error("dangling file remove failed", err) + return + } + q.Log.Printf("removed dangling file %s", name) +} + +func (q *Queue) openMessage(id string) (*QueueMetadata, textproto.Header, buffer.Buffer, error) { + meta, err := q.readMessageMeta(id) + if err != nil { + return nil, textproto.Header{}, nil, err + } + + bodyPath := filepath.Join(q.location, id+".body") + _, err = os.Stat(bodyPath) + if err != nil { + if os.IsNotExist(err) { + q.tryRemoveDanglingFile(id + ".meta") + } + return nil, textproto.Header{}, nil, err + } + body := buffer.FileBuffer{Path: bodyPath} + + headerPath := filepath.Join(q.location, id+".header") + headerFile, err := os.Open(headerPath) + if err != nil { + if os.IsNotExist(err) { + q.tryRemoveDanglingFile(id + ".meta") + q.tryRemoveDanglingFile(id + ".body") + } + return nil, textproto.Header{}, nil, err + } + + bufferedHeader := bufio.NewReader(headerFile) + header, err := textproto.ReadHeader(bufferedHeader) + if err != nil { + return nil, textproto.Header{}, nil, err + } + + return meta, header, body, nil +} + +func (q *Queue) InstanceName() string { + return q.name +} + +func (q *Queue) Name() string { + return "queue" +} + +func (q *Queue) emitDSN(meta *QueueMetadata, header textproto.Header, failedRcpts []string) { + // If, apparently, we have no DSN msgpipeline configured - do nothing. + if q.dsnPipeline == nil { + return + } + + // Null return-path, used in DSNs. + if meta.MsgMeta.OriginalFrom == "" { + return + } + + dsnID, err := module.GenerateMsgID() + if err != nil { + q.Log.Error("rand.Rand error", err) + return + } + + dsnEnvelope := dsn.Envelope{ + MsgID: "<" + dsnID + "@" + q.autogenMsgDomain + ">", + From: "MAILER-DAEMON@" + q.autogenMsgDomain, + To: meta.MsgMeta.OriginalFrom, + } + mtaInfo := dsn.ReportingMTAInfo{ + ReportingMTA: q.hostname, + XSender: meta.From, + XMessageID: meta.MsgMeta.ID, + ArrivalDate: meta.FirstAttempt, + LastAttemptDate: meta.LastAttempt, + } + if !meta.MsgMeta.DontTraceSender && meta.MsgMeta.Conn != nil { + mtaInfo.ReceivedFromMTA = meta.MsgMeta.Conn.Hostname + } + + rcptInfo := make([]dsn.RecipientInfo, 0, len(meta.RcptErrs)) + for _, rcpt := range failedRcpts { + rcptErr := meta.RcptErrs[rcpt] + // rcptErr is stored in RcptErrs using the effective recipient address, + // not the original one. + + originalRcpt := meta.MsgMeta.OriginalRcpts[rcpt] + if originalRcpt != "" { + rcpt = originalRcpt + } + + rcptInfo = append(rcptInfo, dsn.RecipientInfo{ + FinalRecipient: rcpt, + Action: dsn.ActionFailed, + Status: rcptErr.EnhancedCode, + DiagnosticCode: rcptErr, + }) + } + + var dsnBodyBlob bytes.Buffer + dl := target.DeliveryLogger(q.Log, meta.MsgMeta) + dsnHeader, err := dsn.GenerateDSN(meta.MsgMeta.SMTPOpts.UTF8, dsnEnvelope, mtaInfo, rcptInfo, header, &dsnBodyBlob) + if err != nil { + dl.Error("failed to generate fail DSN", err) + return + } + dsnBody := buffer.MemoryBuffer{Slice: dsnBodyBlob.Bytes()} + + dsnMeta := &module.MsgMetadata{ + ID: dsnID, + SMTPOpts: smtp.MailOptions{ + UTF8: meta.MsgMeta.SMTPOpts.UTF8, + RequireTLS: meta.MsgMeta.SMTPOpts.RequireTLS, + }, + } + dl.Msg("generated failed DSN", "dsn_id", dsnID) + + msgCtx, msgTask := trace.NewTask(context.Background(), "DSN Delivery") + defer msgTask.End() + + mailCtx, mailTask := trace.NewTask(msgCtx, "MAIL FROM") + dsnDelivery, err := q.dsnPipeline.Start(mailCtx, dsnMeta, "") + mailTask.End() + if err != nil { + dl.Error("failed to enqueue DSN", err, "dsn_id", dsnID) + return + } + + defer func() { + if err != nil { + dl.Error("failed to enqueue DSN", err, "dsn_id", dsnID) + if err := dsnDelivery.Abort(msgCtx); err != nil { + dl.Error("failed to abort DSN delivery", err, "dsn_id", dsnID) + } + } + }() + + rcptCtx, rcptTask := trace.NewTask(msgCtx, "RCPT TO") + if err = dsnDelivery.AddRcpt(rcptCtx, meta.From, smtp.RcptOptions{}); err != nil { + rcptTask.End() + return + } + rcptTask.End() + + bodyCtx, bodyTask := trace.NewTask(msgCtx, "DATA") + if err = dsnDelivery.Body(bodyCtx, dsnHeader, dsnBody); err != nil { + bodyTask.End() + return + } + if err = dsnDelivery.Commit(bodyCtx); err != nil { + bodyTask.End() + return + } + bodyTask.End() +} + +func init() { + module.Register("target.queue", NewQueue) +} diff --git a/internal/target/queue/queue_test.go b/internal/target/queue/queue_test.go new file mode 100644 index 0000000..ff9a4f6 --- /dev/null +++ b/internal/target/queue/queue_test.go @@ -0,0 +1,828 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package queue + +import ( + "bytes" + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +// newTestQueue returns properly initialized Queue object usable for testing. +// +// See newTestQueueDir to create testing queue from an existing directory. +// It is called responsibility to remove queue directory created by this function. +func newTestQueue(t *testing.T, target module.DeliveryTarget) *Queue { + return newTestQueueDir(t, target, t.TempDir()) +} + +func cleanQueue(t *testing.T, q *Queue) { + t.Log("--- queue.Close") + if err := q.Close(); err != nil { + t.Fatal("queue.Close:", err) + } +} + +func newTestQueueDir(t *testing.T, target module.DeliveryTarget, dir string) *Queue { + mod, _ := NewQueue("", "queue", nil, nil) + q := mod.(*Queue) + q.initialRetryTime = 0 + q.retryTimeScale = 1 + q.postInitDelay = 0 + q.maxTries = 5 + q.location = dir + q.Target = target + + if testing.Verbose() { + q.Log = testutils.Logger(t, "queue") + } else { + q.Log = log.Logger{Out: log.NopOutput{}} + } + + if err := q.start(1); err != nil { + panic(err) + } + + return q +} + +// unreliableTarget is a module.DeliveryTarget implementation that stores +// messages to a slice and sometimes fails with the specified error. +type unreliableTarget struct { + committed chan testutils.Msg + aborted chan testutils.Msg + + // Amount of completed deliveries (both failed and succeeded) + passedMessages int + + // To make unreliableTarget fail Commit for N-th delivery, set N-1-th + // element of this slice to wanted error object. If slice is + // nil/empty or N is bigger than its size - delivery will succeed. + bodyFailures []error + bodyFailuresPartial []map[string]error + rcptFailures []map[string]error +} + +type unreliableTargetDelivery struct { + ut *unreliableTarget + msg testutils.Msg +} + +type unreliableTargetDeliveryPartial struct { + *unreliableTargetDelivery +} + +func (utd *unreliableTargetDelivery) AddRcpt(ctx context.Context, rcptTo string, _ smtp.RcptOptions) error { + if len(utd.ut.rcptFailures) > utd.ut.passedMessages { + rcptErrs := utd.ut.rcptFailures[utd.ut.passedMessages] + if err := rcptErrs[rcptTo]; err != nil { + return err + } + } + + utd.msg.RcptTo = append(utd.msg.RcptTo, rcptTo) + return nil +} + +func (utd *unreliableTargetDelivery) Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error { + if utd.ut.bodyFailuresPartial != nil { + return errors.New("partial failure occurred, no additional information available") + } + + r, _ := body.Open() + utd.msg.Body, _ = io.ReadAll(r) + + if len(utd.ut.bodyFailures) > utd.ut.passedMessages { + return utd.ut.bodyFailures[utd.ut.passedMessages] + } + + return nil +} + +func (utd *unreliableTargetDeliveryPartial) BodyNonAtomic(ctx context.Context, c module.StatusCollector, header textproto.Header, body buffer.Buffer) { + r, _ := body.Open() + utd.msg.Body, _ = io.ReadAll(r) + + if len(utd.ut.bodyFailuresPartial) > utd.ut.passedMessages { + for rcpt, err := range utd.ut.bodyFailuresPartial[utd.ut.passedMessages] { + c.SetStatus(rcpt, err) + } + } +} + +func (utd *unreliableTargetDelivery) Abort(ctx context.Context) error { + utd.ut.passedMessages++ + if utd.ut.aborted != nil { + utd.ut.aborted <- utd.msg + } + return nil +} + +func (utd *unreliableTargetDelivery) Commit(ctx context.Context) error { + utd.ut.passedMessages++ + if utd.ut.committed != nil { + utd.ut.committed <- utd.msg + } + return nil +} + +func (ut *unreliableTarget) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + if ut.bodyFailuresPartial != nil { + return &unreliableTargetDeliveryPartial{ + &unreliableTargetDelivery{ + ut: ut, + msg: testutils.Msg{ + MsgMeta: msgMeta, + MailFrom: mailFrom, + }, + }, + }, nil + } + return &unreliableTargetDelivery{ + ut: ut, + msg: testutils.Msg{ + MsgMeta: msgMeta, + MailFrom: mailFrom, + }, + }, nil +} + +func readMsgChanTimeout(t *testing.T, ch <-chan testutils.Msg, timeout time.Duration) *testutils.Msg { + t.Helper() + timer := time.NewTimer(timeout) + select { + case msg := <-ch: + return &msg + case <-timer.C: + t.Fatal("chan read timed out") + return nil + } +} + +func checkQueueDir(t *testing.T, q *Queue, expectedIDs []string) { + t.Helper() + // We use the map to lookups and also to mark messages we found + // we can report missing entries. + expectedMap := make(map[string]bool, len(expectedIDs)) + for _, id := range expectedIDs { + expectedMap[id] = false + } + + dir, err := os.ReadDir(q.location) + if err != nil { + t.Fatalf("failed to read queue directory: %v", err) + } + + // Queue implementation uses file names in the following format: + // DELIVERY_ID.SOMETHING + for _, file := range dir { + if file.IsDir() { + t.Fatalf("queue should not create subdirectories in the store, but there is %s dir in it", file.Name()) + } + + nameParts := strings.Split(file.Name(), ".") + if len(nameParts) != 2 { + t.Fatalf("did the queue files name format changed? got %s", file.Name()) + } + + _, ok := expectedMap[nameParts[0]] + if !ok { + t.Errorf("message with unexpected Msg ID %s is stored in queue store", nameParts[0]) + continue + } + + expectedMap[nameParts[0]] = true + } + + for id, found := range expectedMap { + if !found { + t.Errorf("expected message with Msg ID %s is missing from queue store", id) + } + } +} + +func TestQueueDelivery(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{committed: make(chan testutils.Msg, 10)} + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // Wait for the delivery to complete and stop processing. + msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) + q.Close() + + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}, "") + + // There should be no queued messages. + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_PermanentFail_NonPartial(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + bodyFailures: []error{ + exterrors.WithTemporary(errors.New("you shall not pass"), false), + }, + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // Queue will abort a delivery if it fails for all recipients. + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + q.Close() + + // Delivery is failed permanently, hence no retry should be rescheduled. + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_PermanentFail_Partial(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + bodyFailuresPartial: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("you shall not pass"), false), + "tester2@example.org": exterrors.WithTemporary(errors.New("you shall not pass"), false), + }, + }, + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // This this is similar to the previous test, but checks PartialDelivery processing logic. + // Here delivery fails for recipients too, but this is reported using PartialDelivery. + + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + q.Close() + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_TemporaryFail(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + bodyFailures: []error{ + exterrors.WithTemporary(errors.New("you shall not pass"), true), + }, + aborted: make(chan testutils.Msg, 10), + committed: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // Delivery should be aborted, because it failed for all recipients. + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + + // Second retry, should work fine. + msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}, "") + + q.Close() + // No more retries scheduled, queue storage is clear. + defer checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_TemporaryFail_Partial(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + bodyFailuresPartial: []map[string]error{ + { + "tester2@example.org": exterrors.WithTemporary(errors.New("go away"), true), + }, + }, + aborted: make(chan testutils.Msg, 10), + committed: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // Committed, tester1@example.org - ok. + msg := readMsgChanTimeout(t, dt.committed, 5000*time.Second) + // Side note: unreliableTarget adds recipients to the msg object even if they were rejected + // later using a partial error. So slice below is all recipients that were submitted by + // the queue. + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}, "") + + // committed #2, tester2@example.org - ok + msg = readMsgChanTimeout(t, dt.committed, 5000*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester2@example.org"}, "") + + q.Close() + // No more retries scheduled, queue storage is clear. + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_MultipleAttempts(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + bodyFailuresPartial: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("you shall not pass 1"), false), + "tester2@example.org": exterrors.WithTemporary(errors.New("you shall not pass 2"), true), + }, + { + "tester2@example.org": exterrors.WithTemporary(errors.New("you shall not pass 3"), true), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org", "tester3@example.org"}) + + // Committed because delivery to tester3@example.org is succeeded. + msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) + // Side note: This slice contains all recipients submitted by the queue, even if + // they were rejected later using partialError. + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org", "tester2@example.org", "tester3@example.org"}, "") + + // tester1 is failed permanently, should not be retried. + // tester2 is failed temporary, should be retried. + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + + // Third attempt... tester2 delivered. + msg = readMsgChanTimeout(t, dt.committed, 5*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester2@example.org"}, "") + + q.Close() + // No more retries should be scheduled. + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_PermanentRcptReject(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), false), + }, + }, + committed: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.org", []string{"tester1@example.org", "tester2@example.org"}) + + // Committed, tester2@example.org succeeded. + msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.org", []string{"tester2@example.org"}, "") + + q.Close() + // No more retries should be scheduled. + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_TemporaryRcptReject(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), true), + }, + }, + committed: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + // First attempt: + // tester1 - temp. fail + // tester2 - ok + // Second attempt: + // tester1 - ok + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) + // Unlike previous tests where unreliableTarget rejected recipients by partialError, here they are rejected + // by AddRcpt directly, so they are NOT saved by the target. + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester2@example.org"}, "") + + msg = readMsgChanTimeout(t, dt.committed, 5*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org"}, "") + + q.Close() + // No more retries should be scheduled. + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_SerializationRoundtrip(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), true), + }, + }, + committed: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + // This is the most tricky test because it is racy and I have no idea what can be done to avoid it. + // It relies on us calling Close before queue msgpipeline decides to retry delivery. + // Hence retry delay is increased from 0ms used in other tests to make it reliable. + q.initialRetryTime = 1 * time.Second + + // To make sure we will not time out due to post-init delay. + q.postInitDelay = 0 + + deliveryID := testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // Standard partial delivery, retry will be scheduled for tester1@example.org. + msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester2@example.org"}, "") + + // Then stop it. + q.Close() + + // Make sure it is saved. + checkQueueDir(t, q, []string{deliveryID}) + + // Then reinit it. + q = newTestQueueDir(t, &dt, q.location) + + // Wait for retry and check it. + msg = readMsgChanTimeout(t, dt.committed, 5*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org"}, "") + + // Close it again. + q.Close() + // No more retries should be scheduled. + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_DeserlizationCleanUp(t *testing.T) { + t.Parallel() + + test := func(t *testing.T, fileSuffix string) { + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), true), + }, + }, + committed: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + // This is the most tricky test because it is racy and I have no idea what can be done to avoid it. + // It relies on us calling Close before queue msgpipeline decides to retry delivery. + // Hence retry delay is increased from 0ms used in other tests to make it reliable. + q.initialRetryTime = 1 * time.Second + + // To make sure we will not time out due to post-init delay. + q.postInitDelay = 0 + + deliveryID := testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // Standard partial delivery, retry will be scheduled for tester1@example.org. + msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester2@example.org"}, "") + + q.Close() + + if err := os.Remove(filepath.Join(q.location, deliveryID+fileSuffix)); err != nil { + t.Fatal(err) + } + + // Dangling files should be removed during load. + q = newTestQueueDir(t, &dt, q.location) + q.Close() + + // Nothing should be left. + checkQueueDir(t, q, []string{}) + } + + t.Run("NoMeta", func(t *testing.T) { + t.Skip("Not implemented") + test(t, ".meta") + }) + t.Run("NoBody", func(t *testing.T) { + test(t, ".body") + }) + t.Run("NoHeader", func(t *testing.T) { + test(t, ".header") + }) +} + +func TestQueueDelivery_AbortIfNoRecipients(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), true), + "tester2@example.org": exterrors.WithTemporary(errors.New("go away"), true), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + readMsgChanTimeout(t, dt.aborted, 5*time.Second) +} + +func TestQueueDelivery_AbortNoDangling(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), true), + "tester2@example.org": exterrors.WithTemporary(errors.New("go away"), true), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + // Copied from testutils.DoTestDelivery. + IDRaw := sha1.Sum([]byte(t.Name())) + encodedID := hex.EncodeToString(IDRaw[:]) + + body := buffer.MemoryBuffer{Slice: []byte("foobar\r\n")} + ctx := module.MsgMetadata{ + DontTraceSender: true, + ID: encodedID, + } + delivery, err := q.Start(context.Background(), &ctx, "test3@example.org") + if err != nil { + t.Fatalf("unexpected Start err: %v", err) + } + for _, rcpt := range [...]string{"test@example.org", "test2@example.org"} { + if err := delivery.AddRcpt(context.Background(), rcpt, smtp.RcptOptions{}); err != nil { + t.Fatalf("unexpected AddRcpt err for %s: %v", rcpt, err) + } + } + if err := delivery.Body(context.Background(), textproto.Header{}, body); err != nil { + t.Fatalf("unexpected Body err: %v", err) + } + if err := delivery.Abort(context.Background()); err != nil { + t.Fatalf("unexpected Abort err: %v", err) + } + + checkQueueDir(t, q, []string{}) +} + +func TestQueueDSN(t *testing.T) { + t.Parallel() + + dsnTarget := unreliableTarget{ + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), false), + "tester2@example.org": exterrors.WithTemporary(errors.New("go away"), false), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + q.hostname = "mx.example.org" + q.autogenMsgDomain = "example.org" + q.dsnPipeline = &dsnTarget + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // Wait for message delivery attempt to complete (aborted because all recipients fail). + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + + // Wait for DSN. + msg := readMsgChanTimeout(t, dsnTarget.committed, 5*time.Second) + + if msg.MailFrom != "" { + t.Fatalf("wrong MAIL FROM address in DSN: %v", msg.MailFrom) + } + if !reflect.DeepEqual(msg.RcptTo, []string{"tester@example.com"}) { + t.Fatalf("wrong RCPT TO address in DSN: %v", msg.RcptTo) + } +} + +func TestQueueDSN_FromEmptyAddr(t *testing.T) { + t.Parallel() + + dsnTarget := unreliableTarget{ + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), false), + "tester2@example.org": exterrors.WithTemporary(errors.New("go away"), false), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + q.hostname = "mx.example.org" + q.autogenMsgDomain = "example.org" + q.dsnPipeline = &dsnTarget + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "", []string{"tester1@example.org", "tester2@example.org"}) + + // Wait for message delivery attempt to complete (aborted because all recipients fail). + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + + time.Sleep(1 * time.Second) + + // There should be no DSN for it. + if dsnTarget.passedMessages != 0 { + t.Errorf("dsnTarget accepted %d messages", dsnTarget.passedMessages) + } + checkQueueDir(t, q, []string{}) +} + +func TestQueueDSN_NoDSNforDSN(t *testing.T) { + t.Parallel() + + dsnTarget := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester@example.org": exterrors.WithTemporary(errors.New("go away"), false), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), false), + "tester2@example.org": exterrors.WithTemporary(errors.New("go away"), false), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + q.hostname = "mx.example.org" + q.autogenMsgDomain = "example.org" + q.dsnPipeline = &dsnTarget + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.org", []string{"tester1@example.org", "tester2@example.org"}) + + // Wait for message delivery attempt to complete (aborted because all recipients fail). + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + + // DSN will be emitted but will fail, so 'aborted' + readMsgChanTimeout(t, dsnTarget.aborted, 5*time.Second) + + time.Sleep(1 * time.Second) + + // There should be no DSN for DSN (dsnTarget handled one message - the DSN itself). + if dsnTarget.passedMessages != 1 { + t.Errorf("dsnTarget accepted %d messages", dsnTarget.passedMessages) + } + checkQueueDir(t, q, []string{}) +} + +func TestQueueDSN_RcptRewrite(t *testing.T) { + t.Parallel() + + dsnTarget := unreliableTarget{ + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "test@example.org": exterrors.WithTemporary(errors.New("go away"), false), + "test2@example.org": exterrors.WithTemporary(errors.New("go away"), false), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + q.hostname = "mx.example.org" + q.autogenMsgDomain = "example.org" + q.dsnPipeline = &dsnTarget + defer cleanQueue(t, q) + + IDRaw := sha1.Sum([]byte(t.Name())) + encodedID := hex.EncodeToString(IDRaw[:]) + + body := buffer.MemoryBuffer{Slice: []byte("foobar\r\n")} + ctx := module.MsgMetadata{ + DontTraceSender: true, + OriginalFrom: "test3@example.org", + OriginalRcpts: map[string]string{ + "test@example.org": "test+public@example.com", + "test2@example.org": "test2+public@example.com", + }, + ID: encodedID, + } + delivery, err := q.Start(context.Background(), &ctx, "test3@example.org") + if err != nil { + t.Fatalf("unexpected Start err: %v", err) + } + for _, rcpt := range [...]string{"test@example.org", "test2@example.org"} { + if err := delivery.AddRcpt(context.Background(), rcpt, smtp.RcptOptions{}); err != nil { + t.Fatalf("unexpected AddRcpt err for %s: %v", rcpt, err) + } + } + if err := delivery.Body(context.Background(), textproto.Header{}, body); err != nil { + t.Fatalf("unexpected Body err: %v", err) + } + if err := delivery.Commit(context.Background()); err != nil { + t.Fatalf("unexpected Commit err: %v", err) + } + + // Wait for message delivery attempt to complete (aborted because all recipients fail). + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + + // Wait for DSN. + msg := readMsgChanTimeout(t, dsnTarget.committed, 5*time.Second) + + if msg.MailFrom != "" { + t.Fatalf("wrong MAIL FROM address in DSN: %v", msg.MailFrom) + } + if !reflect.DeepEqual(msg.RcptTo, []string{"test3@example.org"}) { + t.Fatalf("wrong RCPT TO address in DSN: %v", msg.RcptTo) + } + + if bytes.Contains(msg.Body, []byte("test@example.org")) || bytes.Contains(msg.Body, []byte("test2@example.org")) { + t.Errorf("DSN contents mention real final addresses") + } + if !bytes.Contains(msg.Body, []byte("test+public@example.com")) || !bytes.Contains(msg.Body, []byte("test2+public@example.com")) { + t.Errorf("DSN contents do not mention original addresses") + } +} + +func init() { + dontRecover = true +} diff --git a/internal/target/queue/timewheel.go b/internal/target/queue/timewheel.go new file mode 100644 index 0000000..060804b --- /dev/null +++ b/internal/target/queue/timewheel.go @@ -0,0 +1,146 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package queue + +import ( + "container/list" + "sync" + "sync/atomic" + "time" +) + +type TimeSlot struct { + Time time.Time + Value interface{} +} + +type TimeWheel struct { + stopped uint32 + + slots *list.List + slotsLock sync.Mutex + + updateNotify chan time.Time + stopNotify chan struct{} + + dispatch func(TimeSlot) +} + +func NewTimeWheel(dispatch func(TimeSlot)) *TimeWheel { + tw := &TimeWheel{ + slots: list.New(), + stopNotify: make(chan struct{}), + updateNotify: make(chan time.Time), + dispatch: dispatch, + } + go tw.tick() + return tw +} + +func (tw *TimeWheel) Add(target time.Time, value interface{}) { + if atomic.LoadUint32(&tw.stopped) == 1 { + // Already stopped, ignore. + return + } + + if value == nil { + panic("can't insert nil objects into TimeWheel queue") + } + + tw.slotsLock.Lock() + tw.slots.PushBack(TimeSlot{Time: target, Value: value}) + tw.slotsLock.Unlock() + + tw.updateNotify <- target +} + +func (tw *TimeWheel) Close() { + atomic.StoreUint32(&tw.stopped, 1) + + // Idempotent Close is convenient sometimes. + if tw.stopNotify == nil { + return + } + + tw.stopNotify <- struct{}{} + <-tw.stopNotify + + tw.stopNotify = nil + + close(tw.updateNotify) +} + +func (tw *TimeWheel) tick() { + for { + now := time.Now() + // Look for list element closest to now. + tw.slotsLock.Lock() + var closestSlot TimeSlot + var closestEl *list.Element + for e := tw.slots.Front(); e != nil; e = e.Next() { + slot := e.Value.(TimeSlot) + if slot.Time.Sub(now) < closestSlot.Time.Sub(now) || closestSlot.Value == nil { + closestSlot = slot + closestEl = e + } + } + tw.slotsLock.Unlock() + // Only this goroutine removes elements from TimeWheel so we can be safe using closestSlot. + + // Queue is empty. Just wait until update. + if closestEl == nil { + select { + case <-tw.updateNotify: + continue + case <-tw.stopNotify: + tw.stopNotify <- struct{}{} + return + } + } + + timer := time.NewTimer(closestSlot.Time.Sub(now)) + + selectloop: + for { + select { + case <-timer.C: + tw.slotsLock.Lock() + tw.slots.Remove(closestEl) + tw.slotsLock.Unlock() + + tw.dispatch(closestSlot) + + break selectloop + case newTarget := <-tw.updateNotify: + // Avoid unnecessary restarts if new target is not going to affect our + // current wait time. + if closestSlot.Time.Sub(now) <= newTarget.Sub(now) { + continue + } + + timer.Stop() + // Recalculate new slot time. + break selectloop + case <-tw.stopNotify: + tw.stopNotify <- struct{}{} + return + } + } + } +} diff --git a/internal/target/queue/timewheel_test.go b/internal/target/queue/timewheel_test.go new file mode 100644 index 0000000..d758b60 --- /dev/null +++ b/internal/target/queue/timewheel_test.go @@ -0,0 +1,127 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package queue + +import ( + "testing" + "time" +) + +func TestTimeWheelAdd(t *testing.T) { + t.Parallel() + + called := make(chan TimeSlot) + + w := NewTimeWheel(func(slot TimeSlot) { + called <- slot + }) + defer w.Close() + + w.Add(time.Now().Add(1*time.Second), 1) + + slot := <-called + if val, _ := slot.Value.(int); val != 1 { + t.Errorf("Wrong slot value: %v", slot.Value) + } +} + +func TestTimeWheelAdd_Ordering(t *testing.T) { + t.Parallel() + + called := make(chan TimeSlot) + + w := NewTimeWheel(func(slot TimeSlot) { + called <- slot + }) + defer w.Close() + + w.Add(time.Now().Add(1*time.Second), 1) + w.Add(time.Now().Add(1250*time.Millisecond), 2) + + slot := <-called + if val, _ := slot.Value.(int); val != 1 { + t.Errorf("Wrong first slot value: %v", slot.Value) + } + slot = <-called + if val, _ := slot.Value.(int); val != 2 { + t.Errorf("Wrong second slot value: %v", slot.Value) + } +} + +func TestTimeWheelAdd_Restart(t *testing.T) { + t.Parallel() + + called := make(chan TimeSlot) + + w := NewTimeWheel(func(slot TimeSlot) { + called <- slot + }) + defer w.Close() + + w.Add(time.Now().Add(1*time.Second), 1) + w.Add(time.Now().Add(500*time.Millisecond), 2) + + slot := <-called + if val, _ := slot.Value.(int); val != 2 { + t.Errorf("Wrong first slot value: %v", slot.Value) + } + slot = <-called + if val, _ := slot.Value.(int); val != 1 { + t.Errorf("Wrong second slot value: %v", slot.Value) + } +} + +func TestTimeWheelAdd_MissingGotoBug(t *testing.T) { + t.Parallel() + + called := make(chan TimeSlot) + + w := NewTimeWheel(func(slot TimeSlot) { + called <- slot + }) + defer w.Close() + + w.Add(time.Now().Add(90000*time.Hour), 1) // practically newer + w.Add(time.Now().Add(500*time.Millisecond), 2) // should correctly restart + + slot := <-called + if val, _ := slot.Value.(int); val != 2 { + t.Errorf("Wrong first slot value: %v", slot.Value) + } +} + +func TestTimeWheelAdd_EmptyUpdWait(t *testing.T) { + t.Parallel() + + called := make(chan TimeSlot) + + w := NewTimeWheel(func(slot TimeSlot) { + called <- slot + }) + defer w.Close() + + time.Sleep(500 * time.Millisecond) + + w.Add(time.Now().Add(1*time.Second), 1) + + slot := <-called + if val, _ := slot.Value.(int); val != 1 { + t.Errorf("Wrong slot value: %v", slot.Value) + } +} diff --git a/internal/target/received.go b/internal/target/received.go new file mode 100644 index 0000000..051e5a6 --- /dev/null +++ b/internal/target/received.go @@ -0,0 +1,107 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package target + +import ( + "context" + "errors" + "net" + "strings" + "time" + + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/module" +) + +func SanitizeForHeader(raw string) string { + return strings.Replace(raw, "\n", "", -1) +} + +func GenerateReceived(ctx context.Context, msgMeta *module.MsgMetadata, ourHostname, mailFrom string) (string, error) { + if msgMeta.Conn == nil { + return "", errors.New("can't generate Received for a locally generated message") + } + + builder := strings.Builder{} + + // Empirically guessed value that should be enough to fit + // the entire value in most cases. + builder.Grow(256 + len(msgMeta.Conn.Hostname)) + + if !msgMeta.DontTraceSender && (strings.Contains(msgMeta.Conn.Proto, "SMTP") || + strings.Contains(msgMeta.Conn.Proto, "LMTP")) { + // INTERNATIONALIZATION: See RFC 6531 Section 3.7.3. + hostname, err := dns.SelectIDNA(msgMeta.SMTPOpts.UTF8, msgMeta.Conn.Hostname) + if err == nil { + builder.WriteString("from ") + builder.WriteString(hostname) + } + + if tcpAddr, ok := msgMeta.Conn.RemoteAddr.(*net.TCPAddr); ok { + builder.WriteString(" (") + if msgMeta.Conn.RDNSName != nil { + rdnsName, err := msgMeta.Conn.RDNSName.GetContext(ctx) + if err == nil && rdnsName != nil && rdnsName.(string) != "" { + // INTERNATIONALIZATION: See RFC 6531 Section 3.7.3. + encoded, err := dns.SelectIDNA(msgMeta.SMTPOpts.UTF8, rdnsName.(string)) + if err == nil { + builder.WriteString(encoded) + builder.WriteRune(' ') + } + } + } + builder.WriteRune('[') + builder.WriteString(tcpAddr.IP.String()) + builder.WriteString("])") + } + } + + if ourHostname != "" { + ourHostname, err := dns.SelectIDNA(msgMeta.SMTPOpts.UTF8, ourHostname) + if err == nil { + builder.WriteString(" by ") + builder.WriteString(SanitizeForHeader(ourHostname)) + } + } + + if mailFrom != "" { + // INTERNATIONALIZATION: See RFC 6531 Section 3.7.3. + mailFrom, err := address.SelectIDNA(msgMeta.SMTPOpts.UTF8, mailFrom) + if err == nil { + builder.WriteString(" (envelope-sender <") + builder.WriteString(SanitizeForHeader(mailFrom)) + builder.WriteString(">)") + } + } + + if msgMeta.Conn.Proto != "" { + builder.WriteString(" with ") + if msgMeta.SMTPOpts.UTF8 { + builder.WriteString("UTF8") + } + builder.WriteString(msgMeta.Conn.Proto) + } + builder.WriteString(" id ") + builder.WriteString(msgMeta.ID) + builder.WriteString("; ") + builder.WriteString(time.Now().Format(time.RFC1123Z)) + + return strings.TrimSpace(builder.String()), nil +} diff --git a/internal/target/remote/connect.go b/internal/target/remote/connect.go new file mode 100644 index 0000000..f9d317e --- /dev/null +++ b/internal/target/remote/connect.go @@ -0,0 +1,390 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "context" + "crypto/tls" + "errors" + "net" + "runtime/trace" + "sort" + "time" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/smtpconn" +) + +type mxConn struct { + *smtpconn.C + + // Domain this MX belongs to. + domain string + dnssecOk bool + + // Errors occurred previously on this connection. + errored bool + + reuseLimit int + + // Amount of times connection was used for an SMTP transaction. + transactions int + lastUseAt time.Time + + // MX/TLS security level established for this connection. + mxLevel module.MXLevel + tlsLevel module.TLSLevel +} + +func (c *mxConn) Usable() bool { + if c.C == nil || c.transactions > c.reuseLimit || c.C.Client() == nil || c.errored { + return false + } + return c.C.Client().Reset() == nil +} + +func (c *mxConn) LastUseAt() time.Time { + return c.lastUseAt +} + +func (c *mxConn) Close() error { + return c.C.Close() +} + +func isVerifyError(err error) bool { + var e *tls.CertificateVerificationError + return errors.As(err, &e) +} + +// connect attempts to connect to the MX, first trying STARTTLS with X.509 +// verification but falling back to unauthenticated TLS or plaintext as +// necessary. +// +// Return values: +// - tlsLevel TLS security level that was estabilished. +// - tlsErr Error that prevented TLS from working if tlsLevel != TLSAuthenticated +func (rd *remoteDelivery) connect(ctx context.Context, conn mxConn, host string, tlsCfg *tls.Config) (tlsLevel module.TLSLevel, tlsErr, err error) { + tlsLevel = module.TLSAuthenticated + if rd.rt.tlsConfig != nil { + tlsCfg = rd.rt.tlsConfig.Clone() + tlsCfg.ServerName = host + } + + rd.Log.DebugMsg("trying", "remote_server", host, "domain", conn.domain) + +retry: + // smtpconn.C default TLS behavior is not useful for us, we want to handle + // TLS errors separately hence starttls=false. + _, err = conn.Connect(ctx, config.Endpoint{ + Host: host, + Port: smtpPort, + }, false, nil) + if err != nil { + return module.TLSNone, nil, err + } + + starttlsOk, _ := conn.Client().Extension("STARTTLS") + if starttlsOk && tlsCfg != nil { + if err := conn.Client().StartTLS(tlsCfg); err != nil { + // Here we just issue STARTTLS command. If it fails for some + // reason - this is either a connection problem or server actively + // rejecting STARTTLS (despite advertising STARTTLS). + // We err on the caution side here and do not perform any fallbacks. + conn.DirectClose() + return module.TLSNone, nil, err + } + + // TLS handshake is deferred to here, this is where we check errors and allow fallback. + if err := conn.Client().Hello(rd.rt.hostname); err != nil { + tlsErr = err + + // Attempt TLS without authentication. It is still better than + // plaintext and we might be able to actually authenticate the + // server using DANE-EE/DANE-TA later. + // + // Check tlsLevel is to avoid looping forever if the same verify + // error happens with InsecureSkipVerify too (e.g. certificate is + // *too* broken). + if isVerifyError(err) && tlsLevel == module.TLSAuthenticated { + rd.Log.Error("TLS verify error, trying without authentication", err, "remote_server", host, "domain", conn.domain) + tlsCfg.InsecureSkipVerify = true + tlsLevel = module.TLSEncrypted + + // TODO: Check go-smtp code to make TLS verification errors + // non-sticky so we can properly send QUIT in this case. + conn.DirectClose() + + goto retry + } + + rd.Log.Error("TLS error, trying plaintext", err, "remote_server", host, "domain", conn.domain) + tlsCfg = nil + tlsLevel = module.TLSNone + conn.DirectClose() + + goto retry + } + } else { + tlsLevel = module.TLSNone + } + + return tlsLevel, tlsErr, nil +} + +func (rd *remoteDelivery) attemptMX(ctx context.Context, conn *mxConn, record *net.MX) error { + mxLevel := module.MXNone + + connCtx, cancel := context.WithCancel(ctx) + // Cancel async policy lookups if rd.connect fails. + defer cancel() + + for _, p := range rd.policies { + policyLevel, err := p.CheckMX(connCtx, mxLevel, conn.domain, record.Host, conn.dnssecOk) + if err != nil { + return err + } + if policyLevel > mxLevel { + mxLevel = policyLevel + } + + p.PrepareConn(ctx, record.Host) + } + + tlsLevel, tlsErr, err := rd.connect(connCtx, *conn, record.Host, rd.rt.tlsConfig) + if err != nil { + return err + } + + // Make decision based on the policy and connection state. + // + // Note: All policy errors are marked as temporary to give the local admin + // chance to troubleshoot them without losing messages. + + tlsState, _ := conn.Client().TLSConnectionState() + for _, p := range rd.policies { + policyLevel, err := p.CheckConn(connCtx, mxLevel, tlsLevel, conn.domain, record.Host, tlsState) + if err != nil { + conn.Close() + return exterrors.WithFields(err, map[string]interface{}{"tls_err": tlsErr}) + } + if policyLevel > tlsLevel { + tlsLevel = policyLevel + } + } + + conn.mxLevel = mxLevel + conn.tlsLevel = tlsLevel + + mxLevelCnt.WithLabelValues(rd.rt.Name(), mxLevel.String()).Inc() + tlsLevelCnt.WithLabelValues(rd.rt.Name(), tlsLevel.String()).Inc() + + return nil +} + +func (rd *remoteDelivery) connectionForDomain(ctx context.Context, domain string) (*mxConn, error) { + if c, ok := rd.connections[domain]; ok { + return c, nil + } + + pooledConn, err := rd.rt.pool.Get(ctx, domain) + if err != nil { + return nil, err + } + + var conn *mxConn + // Ignore pool for connections with REQUIRETLS to avoid "pool poisoning" + // where attacker can make messages indeliverable by forcing reuse of old + // connection with weaker security. + if pooledConn != nil && !rd.msgMeta.SMTPOpts.RequireTLS { + conn = pooledConn.(*mxConn) + rd.Log.Msg("reusing cached connection", "domain", domain, "transactions_counter", conn.transactions, + "local_addr", conn.LocalAddr(), "remote_addr", conn.RemoteAddr()) + } else { + rd.Log.DebugMsg("opening new connection", "domain", domain, "cache_ignored", pooledConn != nil) + conn, err = rd.newConn(ctx, domain) + if err != nil { + return nil, err + } + } + + if rd.msgMeta.SMTPOpts.RequireTLS { + if conn.tlsLevel < module.TLSAuthenticated { + conn.Close() + return nil, &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 30}, + Message: "TLS it not available or unauthenticated but required (REQUIRETLS)", + Misc: map[string]interface{}{ + "tls_level": conn.tlsLevel, + }, + } + } + if conn.mxLevel < module.MX_MTASTS { + conn.Close() + return nil, &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 30}, + Message: "Failed to establish the MX record authenticity (REQUIRETLS)", + Misc: map[string]interface{}{ + "mx_level": conn.mxLevel, + }, + } + } + } + + region := trace.StartRegion(ctx, "remote/limits.TakeDest") + if err := rd.rt.limits.TakeDest(ctx, domain); err != nil { + region.End() + conn.Close() + return nil, err + } + region.End() + + // Relaxed REQUIRETLS mode is not conforming to the specification strictly + // but allows to start deploying client support for REQUIRETLS without the + // requirement for servers in the whole world to support it. The assumption + // behind it is that MX for the recipient domain is the final destination + // and all other forwarders behind it already have secure connection to + // each other. Therefore it is enough to enforce strict security only on + // the path to the MX even if it does not support the REQUIRETLS to propagate + // this requirement further. + if ok, _ := conn.Client().Extension("REQUIRETLS"); rd.rt.relaxedREQUIRETLS && !ok { + rd.msgMeta.SMTPOpts.RequireTLS = false + } + + if err := conn.Mail(ctx, rd.mailFrom, rd.msgMeta.SMTPOpts); err != nil { + conn.Close() + return nil, err + } + conn.lastUseAt = time.Now() + + rd.connections[domain] = conn + return conn, nil +} + +func (rd *remoteDelivery) newConn(ctx context.Context, domain string) (*mxConn, error) { + conn := mxConn{ + reuseLimit: rd.rt.connReuseLimit, + C: smtpconn.New(), + domain: domain, + lastUseAt: time.Now(), + } + + conn.Dialer = rd.rt.dialer + conn.Log = rd.Log + conn.Hostname = rd.rt.hostname + conn.AddrInSMTPMsg = true + if rd.rt.connectTimeout != 0 { + conn.ConnectTimeout = rd.rt.connectTimeout + } + if rd.rt.commandTimeout != 0 { + conn.CommandTimeout = rd.rt.commandTimeout + } + if rd.rt.submissionTimeout != 0 { + conn.SubmissionTimeout = rd.rt.submissionTimeout + } + + for _, p := range rd.policies { + p.PrepareDomain(ctx, domain) + } + + region := trace.StartRegion(ctx, "remote/LookupMX") + dnssecOk, records, err := rd.lookupMX(ctx, domain) + region.End() + if err != nil { + return nil, err + } + conn.dnssecOk = dnssecOk + + var lastErr error + region = trace.StartRegion(ctx, "remote/Connect+TLS") + for _, record := range records { + if record.Host == "." { + return nil, &exterrors.SMTPError{ + Code: 556, + EnhancedCode: exterrors.EnhancedCode{5, 1, 10}, + Message: "Domain does not accept email (null MX)", + } + } + + if err := rd.attemptMX(ctx, &conn, record); err != nil { + if len(records) != 0 { + rd.Log.Error("cannot use MX", err, "remote_server", record.Host, "domain", domain) + } + lastErr = err + continue + } + break + } + region.End() + + // Still not connected? Bail out. + if conn.Client() == nil { + return nil, &exterrors.SMTPError{ + Code: exterrors.SMTPCode(lastErr, 451, 550), + EnhancedCode: exterrors.SMTPEnchCode(lastErr, exterrors.EnhancedCode{0, 4, 0}), + Message: "No usable MXs, last err: " + lastErr.Error(), + TargetName: "remote", + Err: lastErr, + Misc: map[string]interface{}{ + "domain": domain, + }, + } + } + + return &conn, nil +} + +func (rd *remoteDelivery) lookupMX(ctx context.Context, domain string) (dnssecOk bool, records []*net.MX, err error) { + if rd.rt.extResolver != nil { + dnssecOk, records, err = rd.rt.extResolver.AuthLookupMX(context.Background(), domain) + } else { + records, err = rd.rt.resolver.LookupMX(ctx, dns.FQDN(domain)) + } + if err != nil { + reason, misc := exterrors.UnwrapDNSErr(err) + return false, nil, &exterrors.SMTPError{ + Code: exterrors.SMTPCode(err, 451, 554), + EnhancedCode: exterrors.SMTPEnchCode(err, exterrors.EnhancedCode{0, 4, 4}), + Message: "MX lookup error", + TargetName: "remote", + Reason: reason, + Err: err, + Misc: misc, + } + } + + sort.Slice(records, func(i, j int) bool { + return records[i].Pref < records[j].Pref + }) + + // Fallback to A/AAA RR when no MX records are present as + // required by RFC 5321 Section 5.1. + if len(records) == 0 { + records = append(records, &net.MX{ + Host: domain, + Pref: 0, + }) + } + + return dnssecOk, records, err +} diff --git a/internal/target/remote/dane.go b/internal/target/remote/dane.go new file mode 100644 index 0000000..2e7dd77 --- /dev/null +++ b/internal/target/remote/dane.go @@ -0,0 +1,158 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "crypto/tls" + "crypto/x509" + "time" + + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" +) + +// Used to override verification time for DANE-TA tests. +var verifyDANETime time.Time + +// verifyDANE checks whether TLSA records require TLS use and match the +// certificate and name used by the server. +// +// overridePKIX result indicates whether DANE should make server authentication +// succeed even if PKIX/X.509 verification fails. That is, if InsecureSkipVerify +// is used and verifyDANE returns overridePKIX=true, the server certificate +// should trusted. +func verifyDANE(recs []dns.TLSA, connState tls.ConnectionState) (overridePKIX bool, err error) { + tlsErr := &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 1}, + Message: "TLS is required but unsupported or failed (enforced by DANE)", + TargetName: "remote", + Misc: map[string]interface{}{ + "remote_server": connState.ServerName, + }, + } + + // See https://tools.ietf.org/html/rfc7672#section-2.2 for requirements of + // TLS discovery. + // We assume upstream resolver will generate an error if the DNSSEC + // signature is bogus so this case is "DNSSEC-authenticated denial of existence". + if len(recs) == 0 { + return false, nil + } + + // Require TLS even if all records are not usable, per Section 2.2 of RFC 7672. + if !connState.HandshakeComplete { + return false, tlsErr + } + + // Ignore invalid records. + var ( + eeRecs []dns.TLSA + taRecs []dns.TLSA + ) + for _, rec := range recs { + switch rec.MatchingType { + case 0, 1, 2: + default: + continue + } + switch rec.Selector { + case 0, 1: + default: + continue + } + + switch rec.Usage { + case 2: + taRecs = append(taRecs, rec) + case 3: + eeRecs = append(eeRecs, rec) + default: + continue + } + } + + // Authentication is not required if all records are unusable, see + // RFC 7672 Section 2.1.1. + if len(eeRecs) == 0 && len(taRecs) == 0 { + return false, nil + } + + for _, rec := range eeRecs { + if rec.Verify(connState.PeerCertificates[0]) == nil { + // https://tools.ietf.org/html/rfc7672#section-3.1.1 + // - SAN/CN are not considered. + // - Expired certificates are fine too. + return true, nil + } + } + + // Don't bother building a temporary certificate pool if there are no + // records to check. + if len(taRecs) == 0 { + return true, &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "No matching TLSA records", + TargetName: "remote", + Misc: map[string]interface{}{ + "remote_server": connState.ServerName, + }, + } + } + + // Collect certificates presented by the server as possible intermediates. + // Add all certificates from the chain that match any record to the root + // pool. + opts := x509.VerifyOptions{ + DNSName: connState.ServerName, + Intermediates: x509.NewCertPool(), + Roots: x509.NewCertPool(), + CurrentTime: verifyDANETime, + } + for _, cert := range connState.PeerCertificates { + root := false + for _, rec := range taRecs { + if cert.IsCA && rec.Verify(cert) == nil { + opts.Roots.AddCert(cert) + root = true + } + } + if !root { + opts.Intermediates.AddCert(cert) + } + } + + // ... then run the standard X.509 verification. This will verify that the + // server certificate chains to any of asserted TA certificates. + if _, err := connState.PeerCertificates[0].Verify(opts); err == nil { + return true, nil + } + + // There are valid records, but none matched. + return false, &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "No matching TLSA records", + TargetName: "remote", + Misc: map[string]interface{}{ + "remote_server": connState.ServerName, + }, + } +} diff --git a/internal/target/remote/dane_delivery_test.go b/internal/target/remote/dane_delivery_test.go new file mode 100644 index 0000000..2b921c7 --- /dev/null +++ b/internal/target/remote/dane_delivery_test.go @@ -0,0 +1,477 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "crypto/tls" + "net" + "strconv" + "testing" + + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" + miekgdns "github.com/miekg/dns" +) + +func targetWithExtResolver(t *testing.T, zones map[string]mockdns.Zone) (*mockdns.Server, *Target) { + dnsSrv, err := mockdns.NewServerWithLogger(zones, testutils.Logger(t, "mockdns"), false) + if err != nil { + t.Fatal(err) + } + + dialer := net.Dialer{} + dialer.Resolver = &net.Resolver{} + dnsSrv.PatchNet(dialer.Resolver) + addr := dnsSrv.LocalAddr().(*net.UDPAddr) + + extResolver, err := dns.NewExtResolver() + if err != nil { + t.Fatal(err) + } + extResolver.Cfg.Servers = []string{addr.IP.String()} + extResolver.Cfg.Port = strconv.Itoa(addr.Port) + + tgt := testTarget(t, zones, extResolver, []module.MXAuthPolicy{ + testDANEPolicy(t, extResolver), + }) + return dnsSrv, tgt +} + +func tlsaRecord(name string, usage, matchType, selector uint8, cert string) map[miekgdns.Type][]miekgdns.RR { + return map[miekgdns.Type][]miekgdns.RR{ + miekgdns.Type(miekgdns.TypeTLSA): { + &miekgdns.TLSA{ + Hdr: miekgdns.RR_Header{ + Name: name, + Class: miekgdns.ClassINET, + Rrtype: miekgdns.TypeTLSA, + Ttl: 9999, + }, + Usage: usage, + MatchingType: matchType, + Selector: selector, + Certificate: cert, + }, + }, + } +} + +func TestRemoteDelivery_DANE_Ok(t *testing.T) { + _, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + // RFC 7672, Section 2.2.2. "Non-CNAME" case. + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.example.invalid.": { + AD: true, + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cf"), + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.policies = append(tgt.policies, + &localPolicy{ + minTLSLevel: module.TLSAuthenticated, // Established via DANE instead of PKIX. + }, + ) + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_DANE_CNAMEd_1(t *testing.T) { + _, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + // RFC 7672, Section 2.2.2. "Secure CNAME" case - TLSA at CNAME matches. + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + CNAME: "mx.cname.invalid.", + }, + "mx.cname.invalid.": { + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.cname.invalid.": { + AD: true, + Misc: tlsaRecord( + "_25._tcp.mx.cname.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cf"), + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.policies = append(tgt.policies, + &localPolicy{ + minTLSLevel: module.TLSAuthenticated, // Established via DANE instead of PKIX. + }, + ) + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_DANE_CNAMEd_2(t *testing.T) { + _, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + // RFC 7672, Section 2.2.2. "Secure CNAME" case - TLSA at initial name matches. + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + CNAME: "mx.cname.invalid.", + }, + "_25._tcp.mx.example.invalid.": { + AD: true, + Misc: tlsaRecord( + "_25._tcp.mx.cname.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cf"), + }, + "mx.cname.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.policies = append(tgt.policies, + &localPolicy{ + minTLSLevel: module.TLSAuthenticated, // Established via DANE instead of PKIX. + }, + ) + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_DANE_InsecureCNAMEDest(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + // RFC 7672, Section 2.2.2. "Insecure CNAME" case - initial name is secure. + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + CNAME: "mx.cname.invalid.", + }, + "_25._tcp.mx.example.invalid.": { + AD: true, + // This is the record that activates DANE but does not match the cert + // => delivery is failed. + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cb"), + }, + "_25._tcp.mx.cname.invalid.": { + AD: false, + // This is the record that matches the cert and would make delivery succeed + // but it should not be considered since AD=false. + Misc: tlsaRecord( + "_25._tcp.mx.cname.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cf"), + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.tlsConfig = clientCfg + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued but should not") + } +} + +func TestRemoteDelivery_DANE_NonAD_TLSA_Ignore(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + // RFC 7672, Section 2.2.2. "Non-CNAME" case - initial name is insecure. + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.example.invalid.": { + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cb"), + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_DANE_NonADIgnore_CNAME(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + // RFC 7672, Section 2.2.2. "Insecure CNAME" case - initial name is insecure. + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + CNAME: "mx.cname.invalid.", + }, + "mx.cname.invalid.": { + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.cname.invalid.": { + AD: true, + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cb"), + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_DANE_SkipAUnauth(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.example.invalid.": { + AD: false, + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "invalid hex will cause serialization error and no response will be sent"), + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.tlsConfig = clientCfg + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_DANE_Mismatch(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.example.invalid.": { + AD: true, + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "ffb5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cf"), + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.tlsConfig = clientCfg + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued but should not") + } +} + +func TestRemoteDelivery_DANE_NoRecord(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.tlsConfig = clientCfg + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_DANE_LookupErr(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.example.invalid.": { + Err: &net.DNSError{}, + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.tlsConfig = clientCfg + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued but should not") + } +} + +func TestRemoteDelivery_DANE_NoTLS(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.example.invalid.": { + AD: true, + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cf"), + }, + } + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued but should not") + } +} + +func TestRemoteDelivery_DANE_TLSError(t *testing.T) { + _, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + AD: true, + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.example.invalid.": { + AD: true, + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cf"), + }, + } + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + + // Cause failure through version incompatibility. + tgt.tlsConfig = &tls.Config{ + MaxVersion: tls.VersionTLS12, + MinVersion: tls.VersionTLS12, + } + srv.TLSConfig.MinVersion = tls.VersionTLS11 + srv.TLSConfig.MaxVersion = tls.VersionTLS11 + + // DANE should prevent the fallback to plaintext. + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued but should not") + } +} diff --git a/internal/target/remote/dane_test.go b/internal/target/remote/dane_test.go new file mode 100644 index 0000000..470fbbb --- /dev/null +++ b/internal/target/remote/dane_test.go @@ -0,0 +1,227 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/pem" + "testing" + "time" + + "github.com/miekg/dns" +) + +// These certificates are related like this: +// +// Root A -> Intermediate A -> Leaf A +// Root B -> LeafB +var ( + rootA = `-----BEGIN CERTIFICATE----- +MIIBMDCB46ADAgECAhRDwag3n5CG90BEO87zEMAPejn6YTAFBgMrZXAwFjEUMBIG +A1UEAxMLVGVzdCBSb290IEEwHhcNMjAxMTI4MjExODA4WhcNMzAxMTI2MjExODA4 +WjAWMRQwEgYDVQQDEwtUZXN0IFJvb3QgQTAqMAUGAytlcAMhADXMzcRec5ocluNR +ExnnNT7I5fmcpjf2P4ik5k0DJNbco0MwQTAPBgNVHRMBAf8EBTADAQH/MA8GA1Ud +DwEB/wQFAwMHBAAwHQYDVR0OBBYEFM5b/b1di1vA+YpMZcsF4K7N1LbaMAUGAytl +cANBAAZ0XTxBDjN9VGPqWjXrYqGPUqbjm4JD3PeHUB4YGH+MNTgeVIlU8qCLIXtM +9kmAkCk7+j5G8p0gMjJMNygeuwE= +-----END CERTIFICATE-----` + intermediateA = `-----BEGIN CERTIFICATE----- +MIIBWjCCAQygAwIBAgIUEOd619/8HC1pWXxaEpQ1vUZOe7wwBQYDK2VwMBYxFDAS +BgNVBAMTC1Rlc3QgUm9vdCBBMB4XDTIwMTEyODIxMTk0M1oXDTMwMTEyNjIxMTk0 +M1owHjEcMBoGA1UEAxMTVGVzdCBJbnRlcm1lZGlhdGUgQTAqMAUGAytlcAMhAFgW +aZz5316olEIHn1Q4RTPd2u/EjN2bo+Cn3EmSlFxto2QwYjAPBgNVHRMBAf8EBTAD +AQH/MA8GA1UdDwEB/wQFAwMHBAAwHQYDVR0OBBYEFB0P00Qphygy+KgkI9tjihFD +ELxhMB8GA1UdIwQYMBaAFM5b/b1di1vA+YpMZcsF4K7N1LbaMAUGAytlcANBAJJH +zsS8ahEjdyRCNUlsPalZiKW8N3G0LnwdVKFhVfcCT+RTRcrMP7vjuWsbJyD5e7hu +z2eCI68xreLQlNySdQ0= +-----END CERTIFICATE-----` + leafA = `-----BEGIN CERTIFICATE----- +MIIBjzCCAUGgAwIBAgIUONvbCs6r9zKFM3IAPRMdrNiJpNgwBQYDK2VwMB4xHDAa +BgNVBAMTE1Rlc3QgSW50ZXJtZWRpYXRlIEEwHhcNMjAxMTI4MjEyMTIyWhcNMzAx +MTI2MjEyMTIyWjAWMRQwEgYDVQQDEwtUZXN0IExlYWYgQTAqMAUGAytlcAMhABIj +W7gwY78RCWHs9eSIdy4x4MXjzdhZwgNSNHHCp5pAo4GYMIGVMAwGA1UdEwEB/wQC +MAAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBUGA1UdEQQOMAyCCm1h +ZGR5LnRlc3QwDwYDVR0PAQH/BAUDAweAADAdBgNVHQ4EFgQU9PFQCnG5fNpNPXUT +8rCuylS6tVwwHwYDVR0jBBgwFoAUHQ/TRCmHKDL4qCQj22OKEUMQvGEwBQYDK2Vw +A0EAGdvHA4VLxpUeUu1Vjom2YX3MukPJG0a3/dB3HiAWWpxMgWfU+Ftie7noaNcI +oUW+M8my46dqN6oXSHU47/QjDg== +-----END CERTIFICATE-----` + rootB = `-----BEGIN CERTIFICATE----- +MIIBMDCB46ADAgECAhRXD7xuPkipDyxyCtm8pZaxhuulaDAFBgMrZXAwFjEUMBIG +A1UEAxMLVGVzdCBSb290IEIwHhcNMjAxMTI4MjExODMwWhcNMzAxMTI2MjExODMw +WjAWMRQwEgYDVQQDEwtUZXN0IFJvb3QgQjAqMAUGAytlcAMhAPOIGJJh5jK8N/Vc +lLrFpysV+SiZjT1Cmt7hoFtMrlbTo0MwQTAPBgNVHRMBAf8EBTADAQH/MA8GA1Ud +DwEB/wQFAwMHBAAwHQYDVR0OBBYEFOLGYf4mkhKbZPwZKCv952tfz/KDMAUGAytl +cANBAOX2gb6ud8CAvOsCgw6uaRm0+jMDVZfkAkNuCIO6cJ/WYfdvuXYXu3e88SuI +gri++h118PomIzJ5PHAaCYsFPgQ= +-----END CERTIFICATE-----` + leafB = `-----BEGIN CERTIFICATE----- +MIIBhzCCATmgAwIBAgIUR2bVQ/Cu4j7Td5TdbWd6Q0LEpOgwBQYDK2VwMBYxFDAS +BgNVBAMTC1Rlc3QgUm9vdCBCMB4XDTIwMTEyODIxMjE0M1oXDTMwMTEyNjIxMjE0 +M1owFjEUMBIGA1UEAxMLVGVzdCBMZWFmIEIwKjAFBgMrZXADIQBiHCTUxF3UxPIV +M/o5OkTtmUrI7AInOvMa0dchU4iJXqOBmDCBlTAMBgNVHRMBAf8EAjAAMB0GA1Ud +JQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAVBgNVHREEDjAMggptYWRkeS50ZXN0 +MA8GA1UdDwEB/wQFAwMHgAAwHQYDVR0OBBYEFPYZPubaAXyr6kXs3khqpMNfdHKK +MB8GA1UdIwQYMBaAFOLGYf4mkhKbZPwZKCv952tfz/KDMAUGAytlcANBABlOwVxE +h7vYmaMYoyOSF1GQiB0ZLsGUjrTNHDnv0+Xp8xG5Td5mGnBi/4Ehq39PdLrj2T7j +3Xy0aiqdDomvwQY= +-----END CERTIFICATE-----` +) + +func parsePEMCert(blob string) *x509.Certificate { + block, _ := pem.Decode([]byte(blob)) + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + panic(err) + } + return cert +} + +func singleTlsaRecord(usage, matchType, selector uint8, cert string) dns.TLSA { + return dns.TLSA{ + Hdr: dns.RR_Header{ + Name: "maddy.test.", + Class: dns.ClassINET, + Rrtype: dns.TypeTLSA, + Ttl: 9999, + }, + Usage: usage, + MatchingType: matchType, + Selector: selector, + Certificate: cert, + } +} + +func keySHA256(blob string) string { + cert := parsePEMCert(blob) + hash := sha256.Sum256(cert.RawSubjectPublicKeyInfo) + return hex.EncodeToString(hash[:]) +} + +func TestVerifyDANE(t *testing.T) { + verifyDANETime = time.Unix(1606600100, 0) + test := func(name string, recs []dns.TLSA, connState tls.ConnectionState, expectErr bool) { + t.Helper() + t.Run(name, func(t *testing.T) { + t.Helper() + _, err := verifyDANE(recs, connState) + if (err != nil) != expectErr { + t.Error("err:", err, "expectErr:", expectErr) + } + }) + } + + // RFC 7672, Section 2.2: + // An "insecure" TLSA RRset or DNSSEC-authenticated denial of existence + // of the TLSA records: + // A connection to the MTA SHOULD be made using (pre-DANE) + // opportunistic TLS; + // + // "Insecure" TLSA RRset results in verifyDANE not being called at all, + // but for the latter (authenticated denial of existence) it is still + // called and should be tested for. + // + // More specific tests for TLSA RRset discovery (including CNAME + // shenanigans) are in dane_delivery_test.go. + test("no TLSA, TLS", []dns.TLSA{}, tls.ConnectionState{ + HandshakeComplete: true, + }, false) + test("no TLSA, no TLS", []dns.TLSA{}, tls.ConnectionState{ + HandshakeComplete: false, + }, false) + + // RFC 7272, Section 2.2: + // A "secure" non-empty TLSA RRset where all the records are unusable: + // Any connection to the MTA MUST be made via TLS, but authentication + // is not required. + test("unusable TLSA, TLS", []dns.TLSA{ + singleTlsaRecord(4, 1, 2, "whatever"), + singleTlsaRecord(4, 5, 2, "whatever"), + singleTlsaRecord(4, 1, 1, "whatever"), + }, tls.ConnectionState{ + HandshakeComplete: true, + PeerCertificates: []*x509.Certificate{parsePEMCert(leafA)}, + }, false) + test("unusable TLSA, no TLS", []dns.TLSA{ + singleTlsaRecord(4, 1, 2, "whatever"), + }, tls.ConnectionState{ + HandshakeComplete: false, + }, true) + + // RFC 7672, Section 2.2: + // A "secure" TLSA RRset with at least one usable record: Any + // connection to the MTA MUST employ TLS encryption and MUST + // authenticate the SMTP server using the techniques discussed in the + // rest of this document. + test("DANE-EE, non-self-signed", []dns.TLSA{ + singleTlsaRecord(3, 1, 1, keySHA256(leafA)), + }, tls.ConnectionState{ + HandshakeComplete: true, + PeerCertificates: []*x509.Certificate{parsePEMCert(leafA)}, + }, false) + test("DANE-EE, multiple records", []dns.TLSA{ + singleTlsaRecord(3, 1, 1, keySHA256(leafB)), + singleTlsaRecord(3, 1, 1, keySHA256(leafA)), + }, tls.ConnectionState{ + HandshakeComplete: true, + PeerCertificates: []*x509.Certificate{parsePEMCert(leafA)}, + }, false) + test("DANE-EE, self-signed", []dns.TLSA{ + singleTlsaRecord(3, 1, 1, keySHA256(rootA)), + }, tls.ConnectionState{ + HandshakeComplete: true, + PeerCertificates: []*x509.Certificate{parsePEMCert(rootA)}, + }, false) + test("DANE-TA, intermediate TA", []dns.TLSA{ + singleTlsaRecord(2, 1, 1, keySHA256(intermediateA)), + }, tls.ConnectionState{ + HandshakeComplete: true, + PeerCertificates: []*x509.Certificate{ + parsePEMCert(leafA), + parsePEMCert(intermediateA), + parsePEMCert(rootA), + }, + }, false) + test("DANE-TA, intermediate TA, mismatch", []dns.TLSA{ + singleTlsaRecord(2, 1, 1, keySHA256(intermediateA)), + }, tls.ConnectionState{ + HandshakeComplete: true, + PeerCertificates: []*x509.Certificate{ + parsePEMCert(leafB), + parsePEMCert(rootB), + }, + }, true) + test("DANE-TA, intermediate TA, multiple records", []dns.TLSA{ + singleTlsaRecord(2, 1, 1, keySHA256(rootB)), + singleTlsaRecord(2, 1, 1, keySHA256(intermediateA)), + // Add multiple times to make sure that multiple records matching the + // same cert do not break anything. + singleTlsaRecord(2, 1, 1, keySHA256(intermediateA)), + }, tls.ConnectionState{ + HandshakeComplete: true, + PeerCertificates: []*x509.Certificate{ + parsePEMCert(leafA), + parsePEMCert(intermediateA), + parsePEMCert(rootA), + }, + }, false) +} diff --git a/internal/target/remote/debugflags.go b/internal/target/remote/debugflags.go new file mode 100644 index 0000000..0a71a10 --- /dev/null +++ b/internal/target/remote/debugflags.go @@ -0,0 +1,35 @@ +//go:build debugflags +// +build debugflags + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + maddycli "github.com/foxcpp/maddy/internal/cli" + "github.com/urfave/cli/v2" +) + +func init() { + maddycli.AddGlobalFlag(&cli.StringFlag{ + Name: "debug.smtpport", + Usage: "SMTP port to use for connections in tests", + Destination: &smtpPort, + }) +} diff --git a/internal/target/remote/metrics.go b/internal/target/remote/metrics.go new file mode 100644 index 0000000..bac88da --- /dev/null +++ b/internal/target/remote/metrics.go @@ -0,0 +1,46 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import "github.com/prometheus/client_golang/prometheus" + +var mxLevelCnt = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "remote", + Name: "conns_mx_level", + Help: "Outbound connections established with specific MX security level", + }, + []string{"module", "level"}, +) + +var tlsLevelCnt = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "remote", + Name: "conns_tls_level", + Help: "Outbound connections established with specific TLS security level", + }, + []string{"module", "level"}, +) + +func init() { + prometheus.MustRegister(mxLevelCnt) + prometheus.MustRegister(tlsLevelCnt) +} diff --git a/internal/target/remote/mxauth_test.go b/internal/target/remote/mxauth_test.go new file mode 100644 index 0000000..2bd8d2d --- /dev/null +++ b/internal/target/remote/mxauth_test.go @@ -0,0 +1,642 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "context" + "crypto/tls" + "errors" + "net" + "strconv" + "testing" + + "github.com/emersion/go-smtp" + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/go-mtasts" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestRemoteDelivery_AuthMX_MTASTS(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + // Testing policy is enough. + Mode: mtasts.ModeTesting, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + }) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_MTASTS_SkipNonMatching(t *testing.T) { + _, be1, srv1 := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.2:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{ + {Host: "mx2.example.invalid.", Pref: 5}, + {Host: "mx1.example.invalid.", Pref: 10}, + }, + }, + "mx1.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "mx2.example.invalid.": { + A: []string{"127.0.0.2"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + Mode: mtasts.ModeEnforce, + MX: []string{"mx2.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + &localPolicy{minMXLevel: module.MX_MTASTS}, + }) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) + + if be1.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_AuthMX_MTASTS_Fail(t *testing.T) { + clientCfg, be1, srv1 := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + Mode: mtasts.ModeTesting, + MX: []string{"mx4.example.invalid"}, // not mx.example.invalid! + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + &localPolicy{minMXLevel: module.MX_MTASTS}, + }) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Fatal("Expected an error, got none") + } + + if be1.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_AuthMX_MTASTS_NoTLS(t *testing.T) { + be1, srv1 := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + Mode: mtasts.ModeEnforce, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + &localPolicy{minMXLevel: module.MX_MTASTS}, + }) + defer tgt.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Fatal("Expected an error, got none") + } + + if be1.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_AuthMX_MTASTS_RequirePKIX(t *testing.T) { + _, be1, srv1 := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + Mode: mtasts.ModeEnforce, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + &localPolicy{minMXLevel: module.MX_MTASTS}, + }) + defer tgt.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Fatal("Expected an error, got none") + } + + if be1.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_AuthMX_MTASTS_NoPolicy(t *testing.T) { + // At the moment, implementation ensures all MX policy checks are completed + // before attempting to connect. + // However, we cannot run complete go-smtp server to check whether it is + // violated and the connection is actually estabilished since this causes + // weird race conditions when test completes before go-smtp has the + // chance to fully initialize itself (Serve is still at the conn.listeners + // assignment when Close is called). + // + // The issue was resolved upstream by introducing locking around internal + // listeners slice use. Uses of FailOnConn remain since they pretty much do + // not hurt. + // + // https://builds.sr.ht/~emersion/job/147975 + tarpit := testutils.FailOnConn(t, "127.0.0.1:"+smtpPort) + defer tarpit.Close() + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return nil, mtasts.ErrNoPolicy + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + &localPolicy{minMXLevel: module.MX_MTASTS}, + }) + defer tgt.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Fatal("Expected an error, got none") + } +} + +func TestRemoteDelivery_AuthMX_DNSSEC(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + AD: true, + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + dnsSrv, err := mockdns.NewServerWithLogger(zones, testutils.Logger(t, "mockdns"), false) + if err != nil { + t.Fatal(err) + } + defer dnsSrv.Close() + + dialer := net.Dialer{} + dialer.Resolver = &net.Resolver{} + dnsSrv.PatchNet(dialer.Resolver) + addr := dnsSrv.LocalAddr().(*net.UDPAddr) + + extResolver, err := dns.NewExtResolver() + if err != nil { + t.Fatal(err) + } + extResolver.Cfg.Servers = []string{addr.IP.String()} + extResolver.Cfg.Port = strconv.Itoa(addr.Port) + + tgt := testTarget(t, zones, extResolver, nil) + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_AuthMX_DNSSEC_Fail(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + dnsSrv, err := mockdns.NewServerWithLogger(zones, testutils.Logger(t, "mockdns"), false) + if err != nil { + t.Fatal(err) + } + defer dnsSrv.Close() + + dialer := net.Dialer{} + dialer.Resolver = &net.Resolver{} + dnsSrv.PatchNet(dialer.Resolver) + addr := dnsSrv.LocalAddr().(*net.UDPAddr) + + extResolver, err := dns.NewExtResolver() + if err != nil { + t.Fatal(err) + } + extResolver.Cfg.Servers = []string{addr.IP.String()} + extResolver.Cfg.Port = strconv.Itoa(addr.Port) + + tgt := testTarget(t, zones, extResolver, []module.MXAuthPolicy{ + &localPolicy{minMXLevel: module.MX_DNSSEC}, + }) + defer tgt.Close() + + _, err = testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Fatal("Expected an error, got none") + } + + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_REQUIRETLS(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + srv.EnableREQUIRETLS = true + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + // Testing policy is enough. + Mode: mtasts.ModeTesting, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + }) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + testutils.DoTestDeliveryMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ + OriginalFrom: "test@example.com", + SMTPOpts: smtp.MailOptions{ + RequireTLS: true, + }, + }) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_REQUIRETLS_Fail(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + srv.EnableREQUIRETLS = false /* no REQUIRETLS */ + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + // Testing policy is enough. + Mode: mtasts.ModeTesting, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + }) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + if _, err := testutils.DoTestDeliveryErrMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ + OriginalFrom: "test@example.com", + SMTPOpts: smtp.MailOptions{ + RequireTLS: true, + }, + }); err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_REQUIRETLS_Relaxed(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + srv.EnableREQUIRETLS = false /* no REQUIRETLS */ + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + // Testing policy is enough. + Mode: mtasts.ModeTesting, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + }) + tgt.relaxedREQUIRETLS = true + tgt.tlsConfig = clientCfg + defer tgt.Close() + + testutils.DoTestDeliveryMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ + OriginalFrom: "test@example.com", + SMTPOpts: smtp.MailOptions{ + RequireTLS: true, + }, + }) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_REQUIRETLS_Relaxed_NoMXAuth(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + srv.EnableREQUIRETLS = false /* no REQUIRETLS */ + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + return nil, mtasts.ErrNoPolicy + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + }) + tgt.relaxedREQUIRETLS = true + tgt.tlsConfig = clientCfg + defer tgt.Close() + + if _, err := testutils.DoTestDeliveryErrMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ + OriginalFrom: "test@example.com", + SMTPOpts: smtp.MailOptions{ + RequireTLS: true, + }, + }); err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_REQUIRETLS_Relaxed_NoTLS(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + srv.EnableREQUIRETLS = false /* no REQUIRETLS */ + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + // Testing policy is enough. + Mode: mtasts.ModeTesting, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + }) + tgt.relaxedREQUIRETLS = true + tgt.tlsConfig = nil + defer tgt.Close() + + if _, err := testutils.DoTestDeliveryErrMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ + OriginalFrom: "test@example.com", + SMTPOpts: smtp.MailOptions{ + RequireTLS: true, + }, + }); err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_REQUIRETLS_Relaxed_TLSFail(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + srv.EnableREQUIRETLS = false /* no REQUIRETLS */ + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + // Testing policy is enough. + Mode: mtasts.ModeTesting, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + }) + tgt.relaxedREQUIRETLS = true + // Cause failure through version incompatibility. + clientCfg.MaxVersion = tls.VersionTLS12 + clientCfg.MinVersion = tls.VersionTLS12 + srv.TLSConfig.MinVersion = tls.VersionTLS11 + srv.TLSConfig.MaxVersion = tls.VersionTLS11 + tgt.tlsConfig = clientCfg + defer tgt.Close() + + if _, err := testutils.DoTestDeliveryErrMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ + OriginalFrom: "test@example.com", + SMTPOpts: smtp.MailOptions{ + RequireTLS: true, + }, + }); err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} diff --git a/internal/target/remote/policy_group.go b/internal/target/remote/policy_group.go new file mode 100644 index 0000000..68a2202 --- /dev/null +++ b/internal/target/remote/policy_group.go @@ -0,0 +1,105 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/module" +) + +// PolicyGroup is a module container for a group of Policy implementations. +// +// It allows to share a set of policy configurations between remote target +// instances using named configuration blocks (module instances) system. +// +// It is registered globally under the name 'mx_auth'. This is also the name of +// corresponding remote target configuration directive. The object does not +// implement any standard module interfaces besides module.Module and is +// specific to the remote target. +type PolicyGroup struct { + L []module.MXAuthPolicy + instName string + pols map[string]module.MXAuthPolicy +} + +func (pg *PolicyGroup) Init(cfg *config.Map) error { + var debugLog bool + cfg.Bool("debug", true, false, &debugLog) + cfg.AllowUnknown() + other, err := cfg.Process() + if err != nil { + return err + } + + // Policies have defined application order since some of them depend on + // results of other policies. We first initialize them in the order they + // are defined in and then reorder depending on the needed order. + + for _, block := range other { + if _, ok := pg.pols[block.Name]; ok { + return config.NodeErr(block, "duplicate policy block: %v", block.Name) + } + + var policy module.MXAuthPolicy + err := modconfig.ModuleFromNode("mx_auth", append([]string{block.Name}, block.Args...), block, cfg.Globals, &policy) + if err != nil { + return err + } + + pg.pols[block.Name] = policy + } + + for _, name := range [...]string{ + "mtasts", + // sts_preload should go after mtasts so it will take not effect if + // MXLevel is already MX_MTASTS. + "sts_preload", + "dane", + "dnssec", + // localPolicy should be the last one, since it considers levels defined by + // other policies. + "local_policy", + } { + policy, ok := pg.pols[name] + if !ok { + continue + } + pg.L = append(pg.L, policy) + } + + return nil +} + +func (PolicyGroup) Name() string { + return "mx_auth" +} + +func (pg PolicyGroup) InstanceName() string { + return pg.instName +} + +func init() { + module.Register("mx_auth", func(_, instName string, _, _ []string) (module.Module, error) { + return &PolicyGroup{ + instName: instName, + pols: map[string]module.MXAuthPolicy{}, + }, nil + }) +} diff --git a/internal/target/remote/remote.go b/internal/target/remote/remote.go new file mode 100644 index 0000000..d4c42ed --- /dev/null +++ b/internal/target/remote/remote.go @@ -0,0 +1,484 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package remote implements module which does outgoing +// message delivery using servers discovered using DNS MX records. +// +// Implemented interfaces: +// - module.DeliveryTarget +package remote + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "runtime/trace" + "strings" + "sync" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + tls2 "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/limits" + "github.com/foxcpp/maddy/internal/smtpconn/pool" + "github.com/foxcpp/maddy/internal/target" + "golang.org/x/net/idna" +) + +var smtpPort = "25" + +func moduleError(err error) error { + return exterrors.WithFields(err, map[string]interface{}{ + "target": "remote", + }) +} + +type Target struct { + name string + hostname string + localIP string + ipv4 bool + tlsConfig *tls.Config + + resolver dns.Resolver + dialer func(ctx context.Context, network, addr string) (net.Conn, error) + extResolver *dns.ExtResolver + + policies []module.MXAuthPolicy + limits *limits.Group + allowSecOverride bool + relaxedREQUIRETLS bool + + pool *pool.P + connReuseLimit int + + Log log.Logger + + connectTimeout time.Duration + commandTimeout time.Duration + submissionTimeout time.Duration +} + +var _ module.DeliveryTarget = &Target{} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, errors.New("remote: inline arguments are not used") + } + // Keep this synchronized with testTarget. + return &Target{ + name: instName, + resolver: dns.DefaultResolver(), + dialer: (&net.Dialer{}).DialContext, + Log: log.Logger{Name: "remote"}, + }, nil +} + +func (rt *Target) Init(cfg *config.Map) error { + var err error + rt.extResolver, err = dns.NewExtResolver() + if err != nil { + rt.Log.Error("cannot initialize DNSSEC-aware resolver, DNSSEC and DANE are not available", err) + } + + cfg.String("hostname", true, true, "", &rt.hostname) + cfg.String("local_ip", false, false, "", &rt.localIP) + cfg.Bool("force_ipv4", false, false, &rt.ipv4) + cfg.Bool("debug", true, false, &rt.Log.Debug) + cfg.Custom("tls_client", true, false, func() (interface{}, error) { + return &tls.Config{}, nil + }, tls2.TLSClientBlock, &rt.tlsConfig) + cfg.Custom("mx_auth", false, false, func() (interface{}, error) { + // Default is "no policies" to follow the principles of explicit + // configuration (if it is not requested - it is not done). + return nil, nil + }, func(cfg *config.Map, n config.Node) (interface{}, error) { + // Module instance is &PolicyGroup. + var p *PolicyGroup + if err := modconfig.GroupFromNode("mx_auth", n.Args, n, cfg.Globals, &p); err != nil { + return nil, err + } + return p.L, nil + }, &rt.policies) + cfg.Custom("limits", false, false, func() (interface{}, error) { + return &limits.Group{}, nil + }, func(cfg *config.Map, n config.Node) (interface{}, error) { + var g *limits.Group + if err := modconfig.GroupFromNode("limits", n.Args, n, cfg.Globals, &g); err != nil { + return nil, err + } + return g, nil + }, &rt.limits) + cfg.Bool("requiretls_override", false, true, &rt.allowSecOverride) + cfg.Bool("relaxed_requiretls", false, true, &rt.relaxedREQUIRETLS) + cfg.Int("conn_reuse_limit", false, false, 10, &rt.connReuseLimit) + cfg.Duration("connect_timeout", false, false, 5*time.Minute, &rt.connectTimeout) + cfg.Duration("command_timeout", false, false, 5*time.Minute, &rt.commandTimeout) + cfg.Duration("submission_timeout", false, false, 5*time.Minute, &rt.submissionTimeout) + + poolCfg := pool.Config{ + MaxKeys: 5000, + MaxConnsPerKey: 5, // basically, max. amount of idle connections in cache + MaxConnLifetimeSec: 150, // 2.5 mins, half of recommended idle time from RFC 5321 + StaleKeyLifetimeSec: 60 * 5, // should be bigger than MaxConnLifetimeSec + } + cfg.Int("conn_max_idle_count", false, false, 5, &poolCfg.MaxConnsPerKey) + cfg.Int64("conn_max_idle_time", false, false, 150, &poolCfg.MaxConnLifetimeSec) + + if _, err := cfg.Process(); err != nil { + return err + } + rt.pool = pool.New(poolCfg) + + // INTERNATIONALIZATION: See RFC 6531 Section 3.7.1. + rt.hostname, err = idna.ToASCII(rt.hostname) + if err != nil { + return fmt.Errorf("remote: cannot represent the hostname as an A-label name: %w", err) + } + + if rt.localIP != "" { + addr, err := net.ResolveTCPAddr("tcp", rt.localIP+":0") + if err != nil { + return fmt.Errorf("remote: failed to parse local IP: %w", err) + } + rt.dialer = (&net.Dialer{ + LocalAddr: addr, + }).DialContext + } + if rt.ipv4 { + dial := rt.dialer + rt.dialer = func(ctx context.Context, network, addr string) (net.Conn, error) { + if network == "tcp" { + network = "tcp4" + } + return dial(ctx, network, addr) + } + } + + return nil +} + +func (rt *Target) Close() error { + rt.pool.Close() + + return nil +} + +func (rt *Target) Name() string { + return "remote" +} + +func (rt *Target) InstanceName() string { + return rt.name +} + +type remoteDelivery struct { + rt *Target + mailFrom string + msgMeta *module.MsgMetadata + Log log.Logger + + recipients []string + connections map[string]*mxConn + + policies []module.DeliveryMXAuthPolicy +} + +func (rt *Target) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + policies := make([]module.DeliveryMXAuthPolicy, 0, len(rt.policies)) + if !(msgMeta.TLSRequireOverride && rt.allowSecOverride) { + for _, p := range rt.policies { + policies = append(policies, p.Start(msgMeta)) + } + } + + var ( + ratelimitDomain string + err error + ) + // This will leave ratelimitDomain = "" for null return path which is fine + // for purposes of ratelimiting. + if mailFrom != "" { + _, ratelimitDomain, err = address.Split(mailFrom) + if err != nil { + return nil, &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 8}, + Message: "Malformed sender address", + TargetName: "remote", + Err: err, + } + } + } + + // Domain is already should be normalized by the message source (e.g. + // endpoint/smtp). + region := trace.StartRegion(ctx, "remote/limits.Take") + addr := net.IPv4(127, 0, 0, 1) + if msgMeta.Conn != nil && msgMeta.Conn.RemoteAddr != nil { + tcpAddr, ok := msgMeta.Conn.RemoteAddr.(*net.TCPAddr) + if ok { + addr = tcpAddr.IP + } + } + if err := rt.limits.TakeMsg(ctx, addr, ratelimitDomain); err != nil { + region.End() + return nil, &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 4, 5}, + Message: "High load, try again later", + Reason: "Global limit timeout", + TargetName: "remote", + Err: err, + } + } + region.End() + + return &remoteDelivery{ + rt: rt, + mailFrom: mailFrom, + msgMeta: msgMeta, + Log: target.DeliveryLogger(rt.Log, msgMeta), + connections: map[string]*mxConn{}, + policies: policies, + }, nil +} + +func (rd *remoteDelivery) AddRcpt(ctx context.Context, to string, opts smtp.RcptOptions) error { + defer trace.StartRegion(ctx, "remote/AddRcpt").End() + + if rd.msgMeta.Quarantine { + return &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Refusing to deliver a quarantined message", + TargetName: "remote", + } + } + + _, domain, err := address.Split(to) + if err != nil { + return err + } + + // Special-case for address. If it is not handled by a rewrite rule before + // - we should not attempt to do anything with it and reject it as invalid. + if domain == "" { + return &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 1, 1}, + Message: " address it no supported", + TargetName: "remote", + } + } + + if strings.HasPrefix(domain, "[") { + return &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 1, 1}, + Message: "IP address literals are not supported", + TargetName: "remote", + } + } + + conn, err := rd.connectionForDomain(ctx, domain) + if err != nil { + return err + } + + if err := conn.Rcpt(ctx, to, opts); err != nil { + return moduleError(err) + } + conn.lastUseAt = time.Now() + + rd.recipients = append(rd.recipients, to) + return nil +} + +type multipleErrs struct { + errs map[string]error + statusLck sync.Mutex +} + +func (m *multipleErrs) Error() string { + m.statusLck.Lock() + defer m.statusLck.Unlock() + return fmt.Sprintf("Partial delivery failure, per-rcpt info: %+v", m.errs) +} + +func (m *multipleErrs) Fields() map[string]interface{} { + m.statusLck.Lock() + defer m.statusLck.Unlock() + + // If there are any temporary errors - the sender should retry to make sure + // all recipients will get the message. However, since we can't tell it + // which recipients got the message, this will generate duplicates for + // them. + // + // We favor delivery with duplicates over incomplete delivery here. + + var ( + code = 550 + enchCode = exterrors.EnhancedCode{5, 0, 0} + ) + for _, err := range m.errs { + if exterrors.IsTemporary(err) { + code = 451 + enchCode = exterrors.EnhancedCode{4, 0, 0} + } + } + + return map[string]interface{}{ + "smtp_code": code, + "smtp_enchcode": enchCode, + "smtp_msg": "Partial delivery failure, additional attempts may result in duplicates", + "target": "remote", + "errs": m.errs, + } +} + +func (m *multipleErrs) SetStatus(rcptTo string, err error) { + m.statusLck.Lock() + defer m.statusLck.Unlock() + m.errs[rcptTo] = err +} + +func (rd *remoteDelivery) Body(ctx context.Context, header textproto.Header, buffer buffer.Buffer) error { + defer trace.StartRegion(ctx, "remote/Body").End() + + merr := multipleErrs{ + errs: make(map[string]error), + } + rd.BodyNonAtomic(ctx, &merr, header, buffer) + + for _, v := range merr.errs { + if v != nil { + if len(merr.errs) == 1 { + return v + } + return &merr + } + } + return nil +} + +func (rd *remoteDelivery) BodyNonAtomic(ctx context.Context, c module.StatusCollector, header textproto.Header, b buffer.Buffer) { + defer trace.StartRegion(ctx, "remote/BodyNonAtomic").End() + + if rd.msgMeta.Quarantine { + for _, rcpt := range rd.recipients { + c.SetStatus(rcpt, &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Refusing to deliver quarantined message", + TargetName: "remote", + }) + } + return + } + + var wg sync.WaitGroup + + for i, conn := range rd.connections { + wg.Add(1) + go func() { + defer wg.Done() + + bodyR, err := b.Open() + if err != nil { + for _, rcpt := range conn.Rcpts() { + c.SetStatus(rcpt, err) + } + return + } + defer bodyR.Close() + + err = conn.Data(ctx, header, bodyR) + for _, rcpt := range conn.Rcpts() { + c.SetStatus(rcpt, err) + } + rd.connections[i].errored = err != nil + conn.lastUseAt = time.Now() + }() + } + + wg.Wait() +} + +func (rd *remoteDelivery) Abort(ctx context.Context) error { + return rd.Close() +} + +func (rd *remoteDelivery) Commit(ctx context.Context) error { + // It is not possible to implement it atomically, so users of remoteDelivery have to + // take care of partial failures. + return rd.Close() +} + +func (rd *remoteDelivery) Close() error { + for _, conn := range rd.connections { + rd.rt.limits.ReleaseDest(conn.domain) + conn.transactions++ + + if !conn.Usable() { + rd.Log.Debugf("disconnected %v from %s (errored=%v,transactions=%v,disconnected before=%v)", + conn.LocalAddr(), conn.ServerName(), conn.errored, conn.transactions, conn.C.Client() == nil) + conn.Close() + } else { + rd.Log.Debugf("returning connection %v for %s to pool", conn.LocalAddr(), conn.ServerName()) + rd.rt.pool.Return(conn.domain, conn) + } + } + + var ( + ratelimitDomain string + err error + ) + if rd.mailFrom != "" { + _, ratelimitDomain, err = address.Split(rd.mailFrom) + if err != nil { + return err + } + } + + addr := net.IPv4(127, 0, 0, 1) + if rd.msgMeta.Conn != nil && rd.msgMeta.Conn.RemoteAddr != nil { + tcpAddr, ok := rd.msgMeta.Conn.RemoteAddr.(*net.TCPAddr) + if ok { + addr = tcpAddr.IP + } + } + rd.rt.limits.ReleaseMsg(addr, ratelimitDomain) + + return nil +} + +func init() { + module.Register("target.remote", New) +} diff --git a/internal/target/remote/remote_test.go b/internal/target/remote/remote_test.go new file mode 100644 index 0000000..4998e0c --- /dev/null +++ b/internal/target/remote/remote_test.go @@ -0,0 +1,1054 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "context" + "crypto/tls" + "flag" + "math/rand" + "net" + "os" + "strconv" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/go-mtasts" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/limits" + "github.com/foxcpp/maddy/internal/smtpconn/pool" + "github.com/foxcpp/maddy/internal/testutils" +) + +// .invalid TLD is used here to make sure if there is something wrong about +// DNS hooks and lookups go to the real Internet, they will not result in +// any useful data that can lead to outgoing connections being made. + +func testTarget(t *testing.T, zones map[string]mockdns.Zone, extResolver *dns.ExtResolver, + extraPolicies []module.MXAuthPolicy) *Target { + resolver := &mockdns.Resolver{Zones: zones} + + tgt := Target{ + name: "remote", + hostname: "mx.example.com", + resolver: resolver, + dialer: resolver.DialContext, + extResolver: extResolver, + tlsConfig: &tls.Config{}, + Log: testutils.Logger(t, "remote"), + policies: extraPolicies, + limits: &limits.Group{}, + pool: pool.New(pool.Config{ + MaxKeys: 5000, + MaxConnsPerKey: 5, // basically, max. amount of idle connections in cache + MaxConnLifetimeSec: 150, // 2.5 mins, half of recommended idle time from RFC 5321 + StaleKeyLifetimeSec: 60 * 5, // should be bigger than MaxConnLifetimeSec + }), + } + + return &tgt +} + +func testSTSPolicy(t *testing.T, zones map[string]mockdns.Zone, mtastsGet func(context.Context, string) (*mtasts.Policy, error)) *mtastsPolicy { + m, err := NewMTASTSPolicy("mx_auth.mtasts", "test", nil, nil) + if err != nil { + t.Fatal(err) + } + p := m.(*mtastsPolicy) + err = p.Init(config.NewMap(nil, config.Node{ + Children: []config.Node{ + { + Name: "cache", + Args: []string{"ram"}, + }, + }, + })) + if err != nil { + t.Fatal(err) + } + + p.mtastsGet = mtastsGet + p.log = testutils.Logger(t, "remote/mtasts") + p.cache.Resolver = &mockdns.Resolver{Zones: zones} + p.StartUpdater() + + return p +} + +func testDANEPolicy(t *testing.T, extR *dns.ExtResolver) *danePolicy { + m, err := NewDANEPolicy("mx_auth.dane", "test", nil, nil) + if err != nil { + t.Fatal(err) + } + p := m.(*danePolicy) + err = p.Init(config.NewMap(nil, config.Node{ + Children: nil, + })) + if err != nil { + t.Fatal(err) + } + + p.extResolver = extR + p.log = testutils.Logger(t, "remote/dane") + return p +} + +func TestRemoteDelivery(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_NoMXFallback(t *testing.T) { + tarpit := testutils.FailOnConn(t, "127.0.0.1:"+smtpPort) + defer tarpit.Close() + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + if err := delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}); err == nil { + t.Fatal("Expected an error, got none") + } + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_EmptySender(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + testutils.DoTestDelivery(t, tgt, "", []string{"test@example.invalid"}) + + be.CheckMsg(t, 0, "", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_IPLiteral(t *testing.T) { + t.Skip("Support disabled") + + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "1.0.0.127.in-addr.arpa.": { + PTR: []string{"mx.example.invalid."}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@[127.0.0.1]"}) + + be.CheckMsg(t, 0, "test@example.com", []string{"test@[127.0.0.1]"}) +} + +func TestRemoteDelivery_FallbackMX(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_BodyNonAtomic(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + c := multipleErrs{ + errs: map[string]error{}, + } + testutils.DoTestDeliveryNonAtomic(t, &c, tgt, "test@example.com", []string{"test@example.invalid"}) + + if err := c.errs["test@example.invalid"]; err != nil { + t.Fatal(err) + } + + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_Abort(t *testing.T) { + _, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + if err := delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_CommitWithoutBody(t *testing.T) { + _, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + if err := delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + // Currently it does nothing, probably it should fail. + if err := delivery.Commit(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_MAILFROMErr(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + be.MailErr = &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + err = delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}) + testutils.CheckSMTPErr(t, err, 550, exterrors.EnhancedCode{5, 1, 2}, "mx.example.invalid. said: Hey") + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_NoMX(t *testing.T) { + tarpit := testutils.FailOnConn(t, "127.0.0.1:"+smtpPort) + defer tarpit.Close() + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + if err := delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}); err == nil { + t.Fatal("Expected an error, got none") + } + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_NullMX(t *testing.T) { + // Hang the test if it actually connects to the server to + // deliver the message. Use of testutils.SMTPServer here + // causes weird race conditions. + tarpit := testutils.FailOnConn(t, "127.0.0.1:"+smtpPort) + defer tarpit.Close() + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: ".", Pref: 10}}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + err = delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}) + testutils.CheckSMTPErr(t, err, 556, exterrors.EnhancedCode{5, 1, 10}, "Domain does not accept email (null MX)") + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_Quarantined(t *testing.T) { + _, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + meta := module.MsgMetadata{ID: "test..."} + + delivery, err := tgt.Start(context.Background(), &meta, "test@example.com") + if err != nil { + t.Fatal(err) + } + + if err := delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + meta.Quarantine = true + + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + body := buffer.MemoryBuffer{Slice: []byte("foobar\n")} + if err := delivery.Body(context.Background(), textproto.Header{}, body); err == nil { + t.Fatal("Expected an error, got none") + } + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_MAILFROMErr_Repeated(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + be.MailErr = &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + err = delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}) + testutils.CheckSMTPErr(t, err, 550, exterrors.EnhancedCode{5, 1, 2}, "mx.example.invalid. said: Hey") + + err = delivery.AddRcpt(context.Background(), "test2@example.invalid", smtp.RcptOptions{}) + testutils.CheckSMTPErr(t, err, 550, exterrors.EnhancedCode{5, 1, 2}, "mx.example.invalid. said: Hey") + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_RcptErr(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + be.RcptErr = map[string]error{ + "test@example.invalid": &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + err = delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}) + testutils.CheckSMTPErr(t, err, 550, exterrors.EnhancedCode{5, 1, 2}, "mx.example.invalid. said: Hey") + + // It should be possible to, however, add another recipient and continue + // delivery as if nothing happened. + if err := delivery.AddRcpt(context.Background(), "test2@example.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + body := buffer.MemoryBuffer{Slice: []byte("foobar\n")} + if err := delivery.Body(context.Background(), hdr, body); err != nil { + t.Fatal(err) + } + + if err := delivery.Commit(context.Background()); err != nil { + t.Fatal(err) + } + + be.CheckMsg(t, 0, "test@example.com", []string{"test2@example.invalid"}) +} + +func TestRemoteDelivery_DownMX(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{ + {Host: "mx1.example.invalid.", Pref: 20}, + {Host: "mx2.example.invalid.", Pref: 10}, + }, + }, + "mx1.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "mx2.example.invalid.": { + A: []string{"127.0.0.2"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_AllMXDown(t *testing.T) { + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{ + {Host: "mx1.example.invalid.", Pref: 20}, + {Host: "mx2.example.invalid.", Pref: 10}, + }, + }, + "mx1.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "mx2.example.invalid.": { + A: []string{"127.0.0.2"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Fatal("Expected an error, got none") + } +} + +func TestRemoteDelivery_Split(t *testing.T) { + be1, srv1 := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + be2, srv2 := testutils.SMTPServer(t, "127.0.0.2:"+smtpPort) + defer srv2.Close() + defer testutils.CheckSMTPConnLeak(t, srv2) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "example2.invalid.": { + MX: []net.MX{{Host: "mx.example2.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "mx.example2.invalid.": { + A: []string{"127.0.0.2"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid", "test@example2.invalid"}) + + be1.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) + be2.CheckMsg(t, 0, "test@example.com", []string{"test@example2.invalid"}) +} + +func TestRemoteDelivery_Split_Fail(t *testing.T) { + be1, srv1 := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + be2, srv2 := testutils.SMTPServer(t, "127.0.0.2:"+smtpPort) + defer srv2.Close() + defer testutils.CheckSMTPConnLeak(t, srv2) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "example2.invalid.": { + MX: []net.MX{{Host: "mx.example2.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "mx.example2.invalid.": { + A: []string{"127.0.0.2"}, + }, + } + + be1.RcptErr = map[string]error{ + "test@example.invalid": &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + err = delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}) + if err == nil { + t.Fatal("Expected an error, got none") + } + + // It should be possible to, however, add another recipient and continue + // delivery as if nothing happened. + if err := delivery.AddRcpt(context.Background(), "test@example2.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + body := buffer.MemoryBuffer{Slice: []byte("foobar\n")} + if err := delivery.Body(context.Background(), hdr, body); err != nil { + t.Fatal(err) + } + + if err := delivery.Commit(context.Background()); err != nil { + t.Fatal(err) + } + + be2.CheckMsg(t, 0, "test@example.com", []string{"test@example2.invalid"}) +} + +func TestRemoteDelivery_BodyErr(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + be.DataErr = &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + err = delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}) + if err != nil { + t.Fatal(err) + } + + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + body := buffer.MemoryBuffer{Slice: []byte("foobar\n")} + if err := delivery.Body(context.Background(), hdr, body); err == nil { + t.Fatal("expected an error, got none") + } + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_Split_BodyErr(t *testing.T) { + be1, srv1 := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + _, srv2 := testutils.SMTPServer(t, "127.0.0.2:"+smtpPort) + defer srv2.Close() + defer testutils.CheckSMTPConnLeak(t, srv2) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "example2.invalid.": { + MX: []net.MX{{Host: "mx.example2.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "mx.example2.invalid.": { + A: []string{"127.0.0.2"}, + }, + } + + be1.DataErr = &smtp.SMTPError{ + Code: 421, + EnhancedCode: smtp.EnhancedCode{4, 1, 2}, + Message: "Hey", + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + if err := delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + if err := delivery.AddRcpt(context.Background(), "test@example2.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + body := buffer.MemoryBuffer{Slice: []byte("foobar\n")} + err = delivery.Body(context.Background(), hdr, body) + testutils.CheckSMTPErr(t, err, 451, exterrors.EnhancedCode{4, 0, 0}, + "Partial delivery failure, additional attempts may result in duplicates") + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_Split_BodyErr_NonAtomic(t *testing.T) { + be1, srv1 := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + _, srv2 := testutils.SMTPServer(t, "127.0.0.2:"+smtpPort) + defer srv2.Close() + defer testutils.CheckSMTPConnLeak(t, srv2) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "example2.invalid.": { + MX: []net.MX{{Host: "mx.example2.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "mx.example2.invalid.": { + A: []string{"127.0.0.2"}, + }, + } + + be1.DataErr = &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + if err := delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + if err := delivery.AddRcpt(context.Background(), "test2@example.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + if err := delivery.AddRcpt(context.Background(), "test@example2.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + body := buffer.MemoryBuffer{Slice: []byte("foobar\n")} + c := multipleErrs{ + errs: map[string]error{}, + } + delivery.(module.PartialDelivery).BodyNonAtomic(context.Background(), &c, hdr, body) + + testutils.CheckSMTPErr(t, c.errs["test@example.invalid"], + 550, exterrors.EnhancedCode{5, 1, 2}, "mx.example.invalid. said: Hey") + testutils.CheckSMTPErr(t, c.errs["test2@example.invalid"], + 550, exterrors.EnhancedCode{5, 1, 2}, "mx.example.invalid. said: Hey") + if err := c.errs["test@example2.invalid"]; err != nil { + t.Errorf("Unexpected error for non-failing connection: %v", err) + } + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_TLSErrFallback(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + // Cause failure through version incompatibility. + clientCfg.MaxVersion = tls.VersionTLS12 + clientCfg.MinVersion = tls.VersionTLS12 + srv.TLSConfig.MinVersion = tls.VersionTLS11 + srv.TLSConfig.MaxVersion = tls.VersionTLS11 + + tgt := testTarget(t, zones, nil, nil) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_RequireTLS_Missing(t *testing.T) { + _, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + &localPolicy{minTLSLevel: module.TLSEncrypted}, + }) + defer tgt.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Errorf("expected an error, got none") + } +} + +func TestRemoteDelivery_RequireTLS_Present(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + &localPolicy{minTLSLevel: module.TLSEncrypted}, + }) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_RequireTLS_NoErrFallback(t *testing.T) { + clientCfg, _, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + // Cause failure through version incompatibility. + clientCfg.MaxVersion = tls.VersionTLS12 + clientCfg.MinVersion = tls.VersionTLS12 + srv.TLSConfig.MinVersion = tls.VersionTLS11 + srv.TLSConfig.MaxVersion = tls.VersionTLS11 + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + &localPolicy{minTLSLevel: module.TLSEncrypted}, + }) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Fatal("Expected an error, got none") + } +} + +func TestRemoteDelivery_TLS_FallbackNoVerify(t *testing.T) { + _, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + // tlsConfig is not configured to trust server cert. + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + &localPolicy{minTLSLevel: module.TLSEncrypted}, + }) + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) + + // But it should still be delivered over TLS. + tlsState, ok := be.Messages[0].Conn.TLSConnectionState() + if !ok || !tlsState.HandshakeComplete { + t.Fatal("Message was not delivered over TLS") + } +} + +func TestRemoteDelivery_TLS_FallbackPlaintext(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + // Cause failure through version incompatibility. + clientCfg.MaxVersion = tls.VersionTLS12 + clientCfg.MinVersion = tls.VersionTLS12 + srv.TLSConfig.MinVersion = tls.VersionTLS11 + srv.TLSConfig.MaxVersion = tls.VersionTLS11 + + tgt := testTarget(t, zones, nil, nil) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestMain(m *testing.M) { + remoteSmtpPort := flag.String("test.smtpport", "random", "(maddy) SMTP port to use for connections in tests") + flag.Parse() + + if *remoteSmtpPort == "random" { + *remoteSmtpPort = strconv.Itoa(rand.Intn(65536-10000) + 10000) + } + + smtpPort = *remoteSmtpPort + os.Exit(m.Run()) +} + +func TestRemoteDelivery_ConnReuse(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + tgt.connReuseLimit = 5 + defer tgt.Close() + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 1, "test@example.com", []string{"test@example.invalid"}) + + if len(be.SourceEndpoints) != 1 { + t.Fatal("Only one session should be used, found", be.SourceEndpoints) + } +} diff --git a/internal/target/remote/security.go b/internal/target/remote/security.go new file mode 100644 index 0000000..a8177fb --- /dev/null +++ b/internal/target/remote/security.go @@ -0,0 +1,642 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "context" + "crypto/tls" + "errors" + "os" + "runtime/debug" + "time" + + "github.com/foxcpp/go-mtasts" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/future" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +type ( + mtastsPolicy struct { + cache *mtasts.Cache + mtastsGet func(context.Context, string) (*mtasts.Policy, error) + updaterStop chan struct{} + log log.Logger + instName string + } + mtastsDelivery struct { + c *mtastsPolicy + domain string + policyFut *future.Future + log log.Logger + } +) + +func NewMTASTSPolicy(_, instName string, _, _ []string) (module.Module, error) { + return &mtastsPolicy{ + instName: instName, + log: log.Logger{Name: "mx_auth.mtasts", Debug: log.DefaultLogger.Debug}, + }, nil +} + +func (c *mtastsPolicy) Name() string { + return c.log.Name +} + +func (c *mtastsPolicy) InstanceName() string { + return c.instName +} + +func (c *mtastsPolicy) Weight() int { + return 10 +} + +func (c *mtastsPolicy) Init(cfg *config.Map) error { + var ( + storeType string + storeDir string + ) + cfg.Enum("cache", false, false, []string{"ram", "fs"}, "fs", &storeType) + cfg.String("fs_dir", false, false, "mtasts_cache", &storeDir) + if _, err := cfg.Process(); err != nil { + return err + } + + switch storeType { + case "fs": + if err := os.MkdirAll(storeDir, os.ModePerm); err != nil { + return err + } + c.cache = mtasts.NewFSCache(storeDir) + case "ram": + c.cache = mtasts.NewRAMCache() + default: + panic("mtasts policy init: unknown cache type") + } + c.cache.Resolver = dns.DefaultResolver() + c.mtastsGet = c.cache.Get + + return nil +} + +// StartUpdater starts a goroutine to update MTA-STS cache periodically until +// Close is called. +// +// It can be called only once per mtastsPolicy instance. +func (c *mtastsPolicy) StartUpdater() { + c.updaterStop = make(chan struct{}) + go c.updater() +} + +func (c *mtastsPolicy) updater() { + defer func() { + if err := recover(); err != nil { + stack := debug.Stack() + log.Printf("panic during MTA-STS update: %v\n%s", err, stack) + log.Printf("MTA-STS cache refresh disabled due to critical error") + c.updaterStop = nil + } + }() + + // Always update cache on start-up since we may have been down for some + // time. + c.log.Debugln("updating MTA-STS cache...") + if err := c.cache.Refresh(); err != nil { + c.log.Error("MTA-STS cache update error", err) + } + c.log.Debugln("updating MTA-STS cache... done!") + + t := time.NewTicker(12 * time.Hour) + for { + select { + case <-t.C: + c.log.Debugln("updating MTA-STS cache...") + if err := c.cache.Refresh(); err != nil { + c.log.Error("MTA-STS cache opdate error", err) + } + c.log.Debugln("updating MTA-STS cache... done!") + case <-c.updaterStop: + c.updaterStop <- struct{}{} + return + } + } +} + +func (c *mtastsPolicy) Start(msgMeta *module.MsgMetadata) module.DeliveryMXAuthPolicy { + return &mtastsDelivery{ + c: c, + log: target.DeliveryLogger(c.log, msgMeta), + } +} + +func (c *mtastsPolicy) Close() error { + if c.updaterStop != nil { + c.updaterStop <- struct{}{} + <-c.updaterStop + c.updaterStop = nil + } + return nil +} + +func (c *mtastsDelivery) PrepareDomain(ctx context.Context, domain string) { + c.policyFut = future.New() + go func() { + c.policyFut.Set(c.c.mtastsGet(ctx, domain)) + }() +} + +func (c *mtastsDelivery) PrepareConn(ctx context.Context, mx string) {} + +func (c *mtastsDelivery) CheckMX(ctx context.Context, mxLevel module.MXLevel, domain, mx string, dnssec bool) (module.MXLevel, error) { + policyI, err := c.policyFut.GetContext(ctx) + if err != nil { + c.log.DebugMsg("MTA-STS error", "err", err) + return module.MXNone, nil + } + policy := policyI.(*mtasts.Policy) + + if !policy.Match(mx) { + if policy.Mode == mtasts.ModeEnforce { + return module.MXNone, &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Failed to establish the MX record authenticity (MTA-STS)", + } + } + c.log.Msg("MX does not match published non-enforced MTA-STS policy", "mx", mx, "domain", c.domain) + return module.MXNone, nil + } + return module.MX_MTASTS, nil +} + +func (c *mtastsDelivery) CheckConn(ctx context.Context, mxLevel module.MXLevel, tlsLevel module.TLSLevel, domain, mx string, tlsState tls.ConnectionState) (module.TLSLevel, error) { + policyI, err := c.policyFut.GetContext(ctx) + if err != nil { + c.c.log.DebugMsg("MTA-STS error", "err", err) + return module.TLSNone, nil + } + policy := policyI.(*mtasts.Policy) + + if policy.Mode != mtasts.ModeEnforce { + return module.TLSNone, nil + } + + if !tlsState.HandshakeComplete { + return module.TLSNone, &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 1}, + Message: "TLS is required but unavailable or failed (MTA-STS)", + } + } + + if tlsState.VerifiedChains == nil { + return module.TLSNone, &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 1}, + Message: "Recipient server TLS certificate is not trusted but " + + "authentication is required by MTA-STS", + Misc: map[string]interface{}{ + "tls_level": tlsLevel, + }, + } + } + + return module.TLSNone, nil +} + +func (c *mtastsDelivery) Reset(msgMeta *module.MsgMetadata) { + c.policyFut = nil + if msgMeta != nil { + c.log = target.DeliveryLogger(c.c.log, msgMeta) + } +} + +// Stub that will be removed in 0.5. +type stsPreloadPolicy struct { + log log.Logger + instName string +} + +func NewSTSPreload(_, instName string, _, _ []string) (module.Module, error) { + return &stsPreloadPolicy{ + instName: instName, + log: log.Logger{Name: "mx_auth.sts_preload", Debug: log.DefaultLogger.Debug}, + }, nil +} + +func (c *stsPreloadPolicy) Name() string { + return c.log.Name +} + +func (c *stsPreloadPolicy) InstanceName() string { + return c.instName +} + +func (c *stsPreloadPolicy) Weight() int { + return 30 // after MTA-STS +} + +func (c *stsPreloadPolicy) Init(cfg *config.Map) error { + c.log.Println("sts_preload module is deprecated and is no-op as the list is expired and unmaintained") + + var ( + sourcePath string + enforceTesting bool + ) + cfg.String("source", false, false, "eff", &sourcePath) + cfg.Bool("enforce_testing", false, true, &enforceTesting) + if _, err := cfg.Process(); err != nil { + return err + } + + return nil +} + +type preloadDelivery struct { + *stsPreloadPolicy +} + +func (p *stsPreloadPolicy) Start(*module.MsgMetadata) module.DeliveryMXAuthPolicy { + return &preloadDelivery{stsPreloadPolicy: p} +} + +func (p *preloadDelivery) Reset(*module.MsgMetadata) {} +func (p *preloadDelivery) PrepareDomain(ctx context.Context, domain string) {} +func (p *preloadDelivery) PrepareConn(ctx context.Context, mx string) {} +func (p *preloadDelivery) CheckMX(ctx context.Context, mxLevel module.MXLevel, domain, mx string, dnssec bool) (module.MXLevel, error) { + return mxLevel, nil +} + +func (p *preloadDelivery) CheckConn(ctx context.Context, mxLevel module.MXLevel, tlsLevel module.TLSLevel, domain, mx string, tlsState tls.ConnectionState) (module.TLSLevel, error) { + return tlsLevel, nil +} + +func (p *stsPreloadPolicy) Close() error { + return nil +} + +type dnssecPolicy struct { + instName string +} + +func NewDNSSECPolicy(_, instName string, _, _ []string) (module.Module, error) { + return &dnssecPolicy{ + instName: instName, + }, nil +} + +func (c *dnssecPolicy) Name() string { + return "mx_auth.dnssec" +} + +func (c *dnssecPolicy) InstanceName() string { + return c.instName +} + +func (c *dnssecPolicy) Weight() int { + return 1 +} + +func (c *dnssecPolicy) Init(cfg *config.Map) error { + _, err := cfg.Process() // will fail if there is any directive + return err +} + +func (dnssecPolicy) Start(*module.MsgMetadata) module.DeliveryMXAuthPolicy { + return dnssecPolicy{} +} + +func (dnssecPolicy) Close() error { + return nil +} + +func (dnssecPolicy) Reset(*module.MsgMetadata) {} +func (dnssecPolicy) PrepareDomain(ctx context.Context, domain string) {} +func (dnssecPolicy) PrepareConn(ctx context.Context, mx string) {} + +func (dnssecPolicy) CheckMX(ctx context.Context, mxLevel module.MXLevel, domain, mx string, dnssec bool) (module.MXLevel, error) { + if dnssec { + return module.MX_DNSSEC, nil + } + return module.MXNone, nil +} + +func (dnssecPolicy) CheckConn(ctx context.Context, mxLevel module.MXLevel, tlsLevel module.TLSLevel, domain, mx string, tlsState tls.ConnectionState) (module.TLSLevel, error) { + return module.TLSNone, nil +} + +type ( + danePolicy struct { + extResolver *dns.ExtResolver + log log.Logger + instName string + } + daneDelivery struct { + c *danePolicy + tlsaFut *future.Future + } +) + +func NewDANEPolicy(_, instName string, _, _ []string) (module.Module, error) { + return &danePolicy{ + instName: instName, + log: log.Logger{Name: "remote/dane", Debug: log.DefaultLogger.Debug}, + }, nil +} + +func (c *danePolicy) Name() string { + return "mx_auth.dane" +} + +func (c *danePolicy) InstanceName() string { + return c.instName +} + +func (c *danePolicy) Weight() int { + return 10 +} + +func (c *danePolicy) Init(cfg *config.Map) error { + var err error + c.extResolver, err = dns.NewExtResolver() + if err != nil { + c.log.Error("DANE support is no-op: unable to init EDNS resolver", err) + } + + cfg.Bool("debug", true, log.DefaultLogger.Debug, &c.log.Debug) + + _, err = cfg.Process() + return err +} + +func (c *danePolicy) Start(*module.MsgMetadata) module.DeliveryMXAuthPolicy { + return &daneDelivery{c: c} +} + +func (c *danePolicy) Close() error { + return nil +} + +func (c *daneDelivery) PrepareDomain(ctx context.Context, domain string) {} + +func (c *daneDelivery) discoverTLSA(ctx context.Context, mx string) ([]dns.TLSA, error) { + adA, rname, err := c.c.extResolver.CheckCNAMEAD(ctx, mx) + if err != nil { + // This may indicate a bogus DNSSEC signature or other lookup issue + // (including non-existing domain). + // Per RFC 7672, any I/O errors (including SERVFAIL) should + // cause delivery to be delayed. + return nil, err + } + if rname == "" { + // No A/AAAA records, short-circuit discovery instead of doing useless + // queries. + return nil, errors.New("no address associated with the host") + } + if !adA { + // If A lookup is not DNSSEC-authenticated we assume the server cannot + // have TLSA record and skip trying to actually lookup TLSA + // to avoid hitting weird errors like SERVFAIL, NOTIMP + // e.g. see https://github.com/foxcpp/maddy/issues/287 + if rname == mx { + c.c.log.Debugln("skipping DANE for", mx, "due to non-authenticated A records") + return nil, nil + } + + // But if it is CNAME'd then we may not want to skip it and actually + // consider initial name since it may be signed. To confirm the + // initial name is signed, do CNAME lookup. + cnameAD, _, err := c.c.extResolver.AuthLookupCNAME(ctx, mx) + if err != nil { + return nil, err + } + if !cnameAD { + c.c.log.Debugln("skipping DANE for", mx, "due to non-authenticated CNAME record") + return nil, nil + } + } + + // If there was a CNAME - try it first. + if rname != mx { + ad, recs, err := c.c.extResolver.AuthLookupTLSA(ctx, "25", "tcp", rname) + if err != nil && !dns.IsNotFound(err) { + return nil, err + } + if ad && len(recs) != 0 { + // recs may be empty or contain only unusable records - this is + // okay per RFC 7672, no fallback to initial name is done. + c.c.log.Debugln("using", len(recs), "DANE records at", rname, "to authenticate", mx) + return recs, nil + } + // Per RFC 7672 Section 2.2 we interpret a non-authenticated RRset just + // like an empty RRset and fallback to trying original name. + c.c.log.Debugln("ignoring non-authenticated TLSA records for", rname) + } + + // If initial name is not a CNAME or final canonical name is not "secure" + // - we consider TLSA under the initial name. + ad, recs, err := c.c.extResolver.AuthLookupTLSA(ctx, "25", "tcp", mx) + if err != nil && !dns.IsNotFound(err) { + return nil, err + } + if !ad { + c.c.log.Debugln("ignoring non-authenticated TLSA records for", mx) + return nil, nil + } + + c.c.log.Debugln("using", len(recs), "DANE records at original name to authenticate", mx) + return recs, nil +} + +func (c *daneDelivery) PrepareConn(ctx context.Context, mx string) { + // No DNSSEC support. + if c.c.extResolver == nil { + return + } + + c.tlsaFut = future.New() + + go func() { + defer func() { + if err := recover(); err != nil { + stack := debug.Stack() + log.Printf("panic during extended resolver lookup: %v\n%s", err, stack) + } + }() + + c.tlsaFut.Set(c.discoverTLSA(ctx, dns.FQDN(mx))) + }() +} + +func (c *daneDelivery) CheckMX(ctx context.Context, mxLevel module.MXLevel, domain, mx string, dnssec bool) (module.MXLevel, error) { + return module.MXNone, nil +} + +func (c *daneDelivery) CheckConn(ctx context.Context, mxLevel module.MXLevel, tlsLevel module.TLSLevel, domain, mx string, tlsState tls.ConnectionState) (module.TLSLevel, error) { + // No DNSSEC support. + if c.c.extResolver == nil { + return module.TLSNone, nil + } + + recsI, err := c.tlsaFut.GetContext(ctx) + if err != nil { + // No records. + if dns.IsNotFound(err) { + return module.TLSNone, nil + } + + // Lookup error here indicates a resolution failure or may also + // indicate a bogus DNSSEC signature. + // There is a big problem with differentiating these two. + // + // We assume DANE failure in both cases as a safety measure. + // However, there is a possibility of a temporary error condition, + // so we mark it as such. + return module.TLSNone, exterrors.WithTemporary(err, true) + } + recs := recsI.([]dns.TLSA) + + overridePKIX, err := verifyDANE(recs, tlsState) + if err != nil { + return module.TLSNone, err + } + if overridePKIX { + return module.TLSAuthenticated, nil + } + return module.TLSNone, nil +} + +func (c *daneDelivery) Reset(*module.MsgMetadata) {} + +type ( + localPolicy struct { + instName string + minTLSLevel module.TLSLevel + minMXLevel module.MXLevel + } +) + +func NewLocalPolicy(_, instName string, _, _ []string) (module.Module, error) { + return &localPolicy{ + instName: instName, + }, nil +} + +func (c *localPolicy) Name() string { + return "mx_auth.local_policy" +} + +func (c *localPolicy) InstanceName() string { + return c.instName +} + +func (c *localPolicy) Weight() int { + return 1000 +} + +func (c *localPolicy) Init(cfg *config.Map) error { + var ( + minTLSLevel string + minMXLevel string + ) + + cfg.Enum("min_tls_level", false, false, + []string{"none", "encrypted", "authenticated"}, "encrypted", &minTLSLevel) + cfg.Enum("min_mx_level", false, false, + []string{"none", "mtasts", "dnssec"}, "none", &minMXLevel) + if _, err := cfg.Process(); err != nil { + return err + } + + // Enum checks the value against allowed list, no 'default' necessary. + switch minTLSLevel { + case "none": + c.minTLSLevel = module.TLSNone + case "encrypted": + c.minTLSLevel = module.TLSEncrypted + case "authenticated": + c.minTLSLevel = module.TLSAuthenticated + } + switch minMXLevel { + case "none": + c.minMXLevel = module.MXNone + case "mtasts": + c.minMXLevel = module.MX_MTASTS + case "dnssec": + c.minMXLevel = module.MX_DNSSEC + } + + return nil +} + +func (l localPolicy) Start(msgMeta *module.MsgMetadata) module.DeliveryMXAuthPolicy { + return l +} + +func (l localPolicy) Close() error { + return nil +} + +func (l localPolicy) Reset(*module.MsgMetadata) {} +func (l localPolicy) PrepareDomain(ctx context.Context, domain string) {} +func (l localPolicy) PrepareConn(ctx context.Context, mx string) {} + +func (l localPolicy) CheckMX(ctx context.Context, mxLevel module.MXLevel, domain, mx string, dnssec bool) (module.MXLevel, error) { + if mxLevel < l.minMXLevel { + return module.MXNone, &exterrors.SMTPError{ + // Err on the side of caution if policy evaluation was messed up by + // a temporary error (we can't know with the current design). + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Failed to establish the MX record authenticity", + Misc: map[string]interface{}{ + "mx_level": mxLevel, + "required_mx_level": l.minMXLevel, + }, + } + } + return module.MXNone, nil +} + +func (l localPolicy) CheckConn(ctx context.Context, mxLevel module.MXLevel, tlsLevel module.TLSLevel, domain, mx string, tlsState tls.ConnectionState) (module.TLSLevel, error) { + if tlsLevel < l.minTLSLevel { + return module.TLSNone, &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 1}, + Message: "TLS it not available or unauthenticated but required", + Misc: map[string]interface{}{ + "tls_level": tlsLevel, + "required_tls_level": l.minTLSLevel, + }, + } + } + return module.TLSNone, nil +} + +func init() { + module.Register("mx_auth.mtasts", NewMTASTSPolicy) + module.Register("mx_auth.sts_preload", NewSTSPreload) + module.Register("mx_auth.dnssec", NewDNSSECPolicy) + module.Register("mx_auth.dane", NewDANEPolicy) + module.Register("mx_auth.local_policy", NewLocalPolicy) +} diff --git a/internal/target/skeleton.go b/internal/target/skeleton.go new file mode 100644 index 0000000..00d481c --- /dev/null +++ b/internal/target/skeleton.go @@ -0,0 +1,130 @@ +//go:build ignore +// +build ignore + +// Copy that file into target/ subdirectory. + +package target_name + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2021 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +const modName = "target.target_name" + +type Target struct { + instName string + log log.Logger +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + // If wanted, extract any values from inlineArgs (these values: + // deliver_to target_name ARG1 ARG2 { ... } + + return &Target{ + instName: instName, + log: log.Logger{Name: instName}, + }, nil +} + +func (t *Target) Init(cfg *config.Map) error { + cfg.Bool("debug", true, false, &t.log.Debug) + + // Read any config directives into Target variables here. + + if _, err := cfg.Process(); err != nil { + return err + } + + // Finish setup using obtained values. + + return nil +} + +func (t *Target) Name() string { + return modName +} + +func (t *Target) InstanceName() string { + return t.instName +} + +// If it necessary to have any server shutdown cleanup - implement Close. + +func (t *Target) Close() error { + return nil +} + +type delivery struct { + t *Target + mailFrom string + log log.Logger + msgMeta *module.MsgMetadata +} + +/* +See module.DeliveryTarget and module.Delivery docs for details on each method. +*/ + +func (t *Target) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + return &delivery{ + t: t, + mailFrom: mailFrom, + log: DeliveryLogger(t.log, msgMeta), + msgMeta: msgMeta, + }, nil +} + +func (d *delivery) AddRcpt(ctx context.Context, rcptTo string) error { + // Corresponds to SMTP RCPT command. + panic("implement me") +} + +func (d *delivery) Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error { + // Corresponds to SMTP DATA command. + panic("implement me") +} + +/* +If Body call can fail partially (either success or fail for each recipient passed to AddRcpt) +- implement BodyNonAtomic and signal status for each recipient using StatusCollector callback. + +func (d *delivery) BodyNonAtomic(ctx context.Context, sc module.StatusCollector, header textproto.Header, body buffer.Buffer) { + +} +*/ + +func (d *delivery) Abort(ctx context.Context) error { + panic("implement me") +} + +func (d *delivery) Commit(ctx context.Context) error { + panic("implement me") +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/target/smtp/sasl.go b/internal/target/smtp/sasl.go new file mode 100644 index 0000000..75f5d4e --- /dev/null +++ b/internal/target/smtp/sasl.go @@ -0,0 +1,77 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp_downstream + +import ( + "github.com/emersion/go-sasl" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" +) + +type saslClientFactory = func(msgMeta *module.MsgMetadata) (sasl.Client, error) + +// saslAuthDirective returns saslClientFactory function used to create sasl.Client. +// for use in outbound connections. +// +// Authentication information of the current client should be passed in arguments. +func saslAuthDirective(_ *config.Map, node config.Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, config.NodeErr(node, "can't declare a block here") + } + if len(node.Args) == 0 { + return nil, config.NodeErr(node, "at least one argument required") + } + switch node.Args[0] { + case "off": + return nil, nil + case "forward": + if len(node.Args) > 1 { + return nil, config.NodeErr(node, "no additional arguments required") + } + return func(msgMeta *module.MsgMetadata) (sasl.Client, error) { + if msgMeta.Conn == nil || msgMeta.Conn.AuthUser == "" || msgMeta.Conn.AuthPassword == "" { + return nil, &exterrors.SMTPError{ + Code: 530, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Authentication is required", + TargetName: "target.smtp", + Reason: "Credentials forwarding is requested but the client is not authenticated", + } + } + return sasl.NewPlainClient("", msgMeta.Conn.AuthUser, msgMeta.Conn.AuthPassword), nil + }, nil + case "plain": + if len(node.Args) != 3 { + return nil, config.NodeErr(node, "two additional arguments are required (username, password)") + } + return func(*module.MsgMetadata) (sasl.Client, error) { + return sasl.NewPlainClient("", node.Args[1], node.Args[2]), nil + }, nil + case "external": + if len(node.Args) > 1 { + return nil, config.NodeErr(node, "no additional arguments required") + } + return func(*module.MsgMetadata) (sasl.Client, error) { + return sasl.NewExternalClient(""), nil + }, nil + default: + return nil, config.NodeErr(node, "unknown authentication mechanism: %s", node.Args[0]) + } +} diff --git a/internal/target/smtp/sasl_test.go b/internal/target/smtp/sasl_test.go new file mode 100644 index 0000000..63b52a5 --- /dev/null +++ b/internal/target/smtp/sasl_test.go @@ -0,0 +1,154 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp_downstream + +import ( + "testing" + + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func testSaslFactory(t *testing.T, args ...string) saslClientFactory { + factory, err := saslAuthDirective(&config.Map{}, config.Node{ + Name: "auth", + Args: args, + }) + if err != nil { + t.Fatal(err) + } + return factory.(saslClientFactory) +} + +func TestSASL_Plain(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + saslFactory: testSaslFactory(t, "plain", "test", "testpass"), + log: testutils.Logger(t, "target.smtp"), + } + + testutils.DoTestDelivery(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + be.CheckMsg(t, 0, "test@example.invalid", []string{"rcpt@example.invalid"}) + if be.Messages[0].AuthUser != "test" { + t.Errorf("Wrong AuthUser: %v", be.Messages[0].AuthUser) + } + if be.Messages[0].AuthPass != "testpass" { + t.Errorf("Wrong AuthPass: %v", be.Messages[0].AuthPass) + } +} + +func TestSASL_Plain_AuthFail(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + be.AuthErr = &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + } + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + saslFactory: testSaslFactory(t, "plain", "test", "testpass"), + log: testutils.Logger(t, "target.smtp"), + } + + _, err := testutils.DoTestDeliveryErr(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } +} + +func TestSASL_Forward(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + saslFactory: testSaslFactory(t, "forward"), + log: testutils.Logger(t, "target.smtp"), + } + + testutils.DoTestDeliveryMeta(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}, &module.MsgMetadata{ + Conn: &module.ConnState{ + AuthUser: "test", + AuthPassword: "testpass", + }, + }) + be.CheckMsg(t, 0, "test@example.invalid", []string{"rcpt@example.invalid"}) + if be.Messages[0].AuthUser != "test" { + t.Errorf("Wrong AuthUser: %v", be.Messages[0].AuthUser) + } + if be.Messages[0].AuthPass != "testpass" { + t.Errorf("Wrong AuthPass: %v", be.Messages[0].AuthPass) + } +} + +func TestSASL_Forward_NoCreds(t *testing.T) { + _, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + saslFactory: testSaslFactory(t, "forward"), + log: testutils.Logger(t, "target.smtp"), + } + + _, err := testutils.DoTestDeliveryErr(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } +} diff --git a/internal/target/smtp/smtp_downstream.go b/internal/target/smtp/smtp_downstream.go new file mode 100644 index 0000000..b85c4ba --- /dev/null +++ b/internal/target/smtp/smtp_downstream.go @@ -0,0 +1,336 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package smtp_downstream provides target.smtp module that implements +// transparent forwarding or messages to configured list of SMTP servers. +// +// Like remote module, this implementation doesn't handle atomic +// delivery properly since it is impossible to do with SMTP protocol +// +// Interfaces implemented: +// - module.DeliveryTarget +package smtp_downstream + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "runtime/trace" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + tls2 "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/smtpconn" + "github.com/foxcpp/maddy/internal/target" + "golang.org/x/net/idna" +) + +type Downstream struct { + modName string + instName string + lmtp bool + targetsArg []string + + starttls bool + hostname string + endpoints []config.Endpoint + saslFactory saslClientFactory + tlsConfig *tls.Config + + connectTimeout time.Duration + commandTimeout time.Duration + submissionTimeout time.Duration + + log log.Logger +} + +func (u *Downstream) moduleError(err error) error { + if err == nil { + return nil + } + + return exterrors.WithFields(err, map[string]interface{}{ + "target": u.modName, + }) +} + +func NewDownstream(modName, instName string, _, inlineArgs []string) (module.Module, error) { + return &Downstream{ + modName: modName, + instName: instName, + lmtp: modName == "target.lmtp" || modName == "lmtp_downstream", /* compatibility with 0.3 configs */ + targetsArg: inlineArgs, + log: log.Logger{Name: modName}, + }, nil +} + +func (u *Downstream) Init(cfg *config.Map) error { + var attemptTLS *bool + + var targetsArg []string + cfg.Bool("debug", true, false, &u.log.Debug) + cfg.Callback("require_tls", func(m *config.Map, node config.Node) error { + u.log.Msg("require_tls directive is deprecated and ignored") + return nil + }) + cfg.Callback("attempt_starttls", func(m *config.Map, node config.Node) error { + u.log.Msg("attempt_starttls directive is deprecated and equivalent to starttls") + + if len(node.Args) == 0 { + trueVal := true + attemptTLS = &trueVal + return nil + } + if len(node.Args) != 1 { + return config.NodeErr(node, "expected exactly 1 argument") + } + + b, err := config.ParseBool(node.Args[0]) + if err != nil { + return err + } + attemptTLS = &b + return nil + }) + cfg.Bool("starttls", false, !u.lmtp, &u.starttls) + cfg.String("hostname", true, true, "", &u.hostname) + cfg.StringList("targets", false, false, nil, &targetsArg) + cfg.Custom("auth", false, false, func() (interface{}, error) { + return nil, nil + }, saslAuthDirective, &u.saslFactory) + cfg.Custom("tls_client", true, false, func() (interface{}, error) { + return &tls.Config{}, nil + }, tls2.TLSClientBlock, &u.tlsConfig) + cfg.Duration("connect_timeout", false, false, 5*time.Minute, &u.connectTimeout) + cfg.Duration("command_timeout", false, false, 5*time.Minute, &u.commandTimeout) + cfg.Duration("submission_timeout", false, false, 5*time.Minute, &u.submissionTimeout) + + if _, err := cfg.Process(); err != nil { + return err + } + + if attemptTLS != nil { + u.starttls = *attemptTLS + } + + // INTERNATIONALIZATION: See RFC 6531 Section 3.7.1. + var err error + u.hostname, err = idna.ToASCII(u.hostname) + if err != nil { + return fmt.Errorf("%s: cannot represent the hostname as an A-label name: %w", u.modName, err) + } + + u.targetsArg = append(u.targetsArg, targetsArg...) + for _, tgt := range u.targetsArg { + endp, err := config.ParseEndpoint(tgt) + if err != nil { + return err + } + + u.endpoints = append(u.endpoints, endp) + } + + if len(u.endpoints) == 0 { + return fmt.Errorf("%s: at least one target endpoint is required", u.modName) + } + + return nil +} + +func (u *Downstream) Name() string { + return u.modName +} + +func (u *Downstream) InstanceName() string { + return u.instName +} + +type delivery struct { + u *Downstream + log log.Logger + + msgMeta *module.MsgMetadata + mailFrom string + rcpts []string + + conn *smtpconn.C +} + +// lmtpDelivery implements module.PartialDelivery +type lmtpDelivery struct { + *delivery +} + +func (u *Downstream) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + defer trace.StartRegion(ctx, "target.smtp/Start").End() + + d := &delivery{ + u: u, + log: target.DeliveryLogger(u.log, msgMeta), + msgMeta: msgMeta, + mailFrom: mailFrom, + } + if err := d.connect(ctx); err != nil { + return nil, err + } + + if err := d.conn.Mail(ctx, mailFrom, msgMeta.SMTPOpts); err != nil { + d.conn.Close() + return nil, err + } + + if u.lmtp { + return &lmtpDelivery{delivery: d}, nil + } + + return d, nil +} + +func (d *delivery) connect(ctx context.Context) error { + // TODO: Review possibility of connection pooling here. + var lastErr error + + conn := smtpconn.New() + conn.Log = d.log + conn.Hostname = d.u.hostname + conn.AddrInSMTPMsg = false + if d.u.connectTimeout != 0 { + conn.ConnectTimeout = d.u.connectTimeout + } + if d.u.commandTimeout != 0 { + conn.CommandTimeout = d.u.commandTimeout + } + if d.u.submissionTimeout != 0 { + conn.SubmissionTimeout = d.u.submissionTimeout + } + + for _, endp := range d.u.endpoints { + var err error + if d.u.lmtp { + _, err = conn.ConnectLMTP(ctx, endp, d.u.starttls, d.u.tlsConfig) + } else { + _, err = conn.Connect(ctx, endp, d.u.starttls, d.u.tlsConfig) + } + if err != nil { + if len(d.u.endpoints) != 1 { + d.log.Msg("connect error", err, "downstream_server", net.JoinHostPort(endp.Host, endp.Port)) + } + lastErr = err + continue + } + + d.log.DebugMsg("connected", "downstream_server", conn.ServerName()) + + lastErr = nil + break + } + if lastErr != nil { + return d.u.moduleError(lastErr) + } + + if d.u.saslFactory != nil { + saslClient, err := d.u.saslFactory(d.msgMeta) + if err != nil { + conn.Close() + return err + } + + if err := conn.Client().Auth(saslClient); err != nil { + conn.Close() + return err + } + } + + d.conn = conn + + return nil +} + +func (d *delivery) AddRcpt(ctx context.Context, rcptTo string, opts smtp.RcptOptions) error { + err := d.conn.Rcpt(ctx, rcptTo, opts) + if err != nil { + return d.u.moduleError(err) + } + + d.rcpts = append(d.rcpts, rcptTo) + return nil +} + +func (d *delivery) Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error { + r, err := body.Open() + if err != nil { + return exterrors.WithFields(err, map[string]interface{}{"target": d.u.modName}) + } + + defer r.Close() + return d.u.moduleError(d.conn.Data(ctx, header, r)) +} + +func (d *lmtpDelivery) BodyNonAtomic(ctx context.Context, sc module.StatusCollector, header textproto.Header, body buffer.Buffer) { + r, err := body.Open() + if err != nil { + modErr := d.u.moduleError(err) + for _, rcpt := range d.rcpts { + sc.SetStatus(rcpt, modErr) + } + } + defer r.Close() + + rcptIndx := 0 + err = d.conn.LMTPData(ctx, header, r, func(rcpt string, err *smtp.SMTPError) { + if err == nil { + sc.SetStatus(rcpt, nil) + } else { + sc.SetStatus(rcpt, &exterrors.SMTPError{ + Code: err.Code, + EnhancedCode: exterrors.EnhancedCode(err.EnhancedCode), + Message: err.Message, + TargetName: d.u.modName, + Err: err, + }) + } + rcptIndx++ + }) + if err != nil { + modErr := d.u.moduleError(err) + for _, rcpt := range d.rcpts[rcptIndx:] { + sc.SetStatus(rcpt, modErr) + } + } +} + +func (d *delivery) Abort(ctx context.Context) error { + d.conn.Close() + return nil +} + +func (d *delivery) Commit(ctx context.Context) error { + return d.conn.Close() +} + +func init() { + module.Register("target.smtp", NewDownstream) + module.Register("target.lmtp", NewDownstream) +} diff --git a/internal/target/smtp/smtp_downstream_test.go b/internal/target/smtp/smtp_downstream_test.go new file mode 100644 index 0000000..f0f58c6 --- /dev/null +++ b/internal/target/smtp/smtp_downstream_test.go @@ -0,0 +1,272 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp_downstream + +import ( + "errors" + "flag" + "math/rand" + "os" + "strconv" + "testing" + + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/internal/testutils" +) + +var testPort string + +func TestDownstreamDelivery(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + tarpit := testutils.FailOnConn(t, "127.0.0.2:"+testPort) + defer tarpit.Close() + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + { + Scheme: "tcp", + Host: "127.0.0.2", + Port: testPort, + }, + }, + log: testutils.Logger(t, "target.smtp"), + } + + testutils.DoTestDelivery(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + be.CheckMsg(t, 0, "test@example.invalid", []string{"rcpt@example.invalid"}) +} + +func TestDownstreamDelivery_LMTP(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort, func(srv *smtp.Server) { + srv.LMTP = true + }) + be.LMTPDataErr = []error{ + nil, + &smtp.SMTPError{ + Code: 501, + Message: "nop", + }, + } + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + modName: "target.lmtp", + lmtp: true, + log: testutils.Logger(t, "lmtp_downstream"), + } + + sc := make(statusCollector) + + testutils.DoTestDeliveryNonAtomic(t, &sc, mod, "test@example.invalid", []string{"rcpt1@example.invalid", "rcpt2@example.invalid"}) + be.CheckMsg(t, 0, "test@example.invalid", []string{"rcpt1@example.invalid", "rcpt2@example.invalid"}) + + if len(sc) != 2 { + t.Fatal("Two statuses should be set") + } + if err := sc["rcpt1@example.invalid"]; err != nil { + t.Fatal("Unexpected error for rcpt1:", err) + } + if sc["rcpt2@example.invalid"] == nil { + t.Fatal("Expected an error for rcpt2") + } + var rcptErr *exterrors.SMTPError + if !errors.As(sc["rcpt2@example.invalid"], &rcptErr) { + t.Fatalf("Not SMTPError: %T", rcptErr) + } + if rcptErr.Code != 501 { + t.Fatal("Wrong SMTP code:", rcptErr.Code) + } +} + +func TestDownstreamDelivery_LMTP_ErrorCoerce(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort, func(srv *smtp.Server) { + srv.LMTP = true + }) + be.LMTPDataErr = []error{ + nil, + &smtp.SMTPError{ + Code: 501, + Message: "nop", + }, + } + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + modName: "target.lmtp", + lmtp: true, + log: testutils.Logger(t, "lmtp_downstream"), + } + + _, err := testutils.DoTestDeliveryErr(t, mod, "test@example.invalid", []string{"rcpt1@example.invalid", "rcpt2@example.invalid"}) + if err == nil { + t.Error("expected failure") + } +} + +type statusCollector map[string]error + +func (sc *statusCollector) SetStatus(rcptTo string, err error) { + (*sc)[rcptTo] = err +} + +func TestDownstreamDelivery_Fallback(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.2:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + { + Scheme: "tcp", + Host: "127.0.0.2", + Port: testPort, + }, + }, + log: testutils.Logger(t, "target.smtp"), + } + + testutils.DoTestDelivery(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + be.CheckMsg(t, 0, "test@example.invalid", []string{"rcpt@example.invalid"}) +} + +func TestDownstreamDelivery_MAILErr(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + be.MailErr = &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + } + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + log: testutils.Logger(t, "target.smtp"), + } + + _, err := testutils.DoTestDeliveryErr(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + testutils.CheckSMTPErr(t, err, 550, exterrors.EnhancedCode{5, 1, 2}, "Hey") +} + +func TestDownstreamDelivery_StartTLS(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + tlsConfig: clientCfg.Clone(), + starttls: true, + log: testutils.Logger(t, "target.smtp"), + } + + testutils.DoTestDelivery(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + be.CheckMsg(t, 0, "test@example.invalid", []string{"rcpt@example.invalid"}) + + tlsState, ok := be.Messages[0].Conn.TLSConnectionState() + if !ok || !tlsState.HandshakeComplete { + t.Fatal("Message was not delivered over TLS") + } +} + +func TestDownstreamDelivery_StartTLS_NoFallback(t *testing.T) { + _, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + starttls: true, + log: testutils.Logger(t, "target.smtp"), + } + + _, err := testutils.DoTestDeliveryErr(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } +} + +func TestMain(m *testing.M) { + remoteSmtpPort := flag.String("test.smtpport", "random", "(maddy) SMTP port to use for connections in tests") + flag.Parse() + + if *remoteSmtpPort == "random" { + *remoteSmtpPort = strconv.Itoa(rand.Intn(65536-10000) + 10000) + } + + testPort = *remoteSmtpPort + os.Exit(m.Run()) +} diff --git a/internal/target/smtp/smtputf8_test.go b/internal/target/smtp/smtputf8_test.go new file mode 100644 index 0000000..74aae23 --- /dev/null +++ b/internal/target/smtp/smtputf8_test.go @@ -0,0 +1,61 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp_downstream + +import ( + "testing" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestDownstreamDelivery_EHLO_ALabel(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod, err := NewDownstream("", "", nil, []string{"tcp://127.0.0.1:" + testPort}) + if err != nil { + t.Fatal(err) + } + if err := mod.Init(config.NewMap(nil, config.Node{ + Children: []config.Node{ + { + Name: "hostname", + Args: []string{"тест.invalid"}, + }, + { + Name: "starttls", + Args: []string{"no"}, + }, + }, + })); err != nil { + t.Fatal(err) + } + + tgt := mod.(*Downstream) + tgt.log = testutils.Logger(t, "remote") + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) + if be.Messages[0].Conn.Hostname() != "xn--e1aybc.invalid" { + t.Error("target/remote should use use Punycode in EHLO") + } +} diff --git a/internal/testutils/bench_delivery.go b/internal/testutils/bench_delivery.go new file mode 100644 index 0000000..f434efc --- /dev/null +++ b/internal/testutils/bench_delivery.go @@ -0,0 +1,141 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "bufio" + "context" + "crypto/sha1" + "encoding/hex" + "io" + "strconv" + "strings" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/module" +) + +// Empirically observed "around average" values. +const ( + MessageBodySize = 100 * 1024 + ExtraMessageHeaderFields = 10 + ExtraMessageHeaderFieldSize = 50 +) + +const testHeaderString = "Content-Type: multipart/mixed; boundary=message-boundary\r\n" + + "Date: Sat, 19 Jun 2016 12:00:00 +0900\r\n" + + "From: Mitsuha Miyamizu \r\n" + + "Reply-To: Mitsuha Miyamizu \r\n" + + "Message-Id: 42@example.org\r\n" + + "MIME-Version: 1.0\r\n" + + "Content-Transfer-Encoding: 8but\r\n" + + "Subject: Your Name.\r\n" + + "To: Taki Tachibana \r\n" + + "\r\n" + +const testAltHeaderString = "Content-Type: multipart/alternative; boundary=b2\r\n" + + "\r\n" + +const testTextHeaderString = "Content-Disposition: inline\r\n" + + "Content-Type: text/plain\r\n" + + "\r\n" + +const testTextBodyString = "What's your name?" + +const testTextString = testTextHeaderString + testTextBodyString + +const testHTMLHeaderString = "Content-Disposition: inline\r\n" + + "Content-Type: text/html\r\n" + + "\r\n" + +const testHTMLBodyString = "
What's your name?
" + +const testHTMLString = testHTMLHeaderString + testHTMLBodyString + +const testAttachmentHeaderString = "Content-Disposition: attachment; filename=note.txt\r\n" + + "Content-Type: text/plain\r\n" + + "\r\n" + +const testAttachmentBodyString = "My name is Mitsuha." + +const testAttachmentString = testAttachmentHeaderString + testAttachmentBodyString + +const testBodyString = "--message-boundary\r\n" + + testAltHeaderString + + "\r\n--b2\r\n" + + testTextString + + "\r\n--b2\r\n" + + testHTMLString + + "\r\n--b2--\r\n" + + "\r\n--message-boundary\r\n" + + testAttachmentString + + "\r\n--message-boundary--\r\n" + +var testMailString = testHeaderString + testBodyString + strings.Repeat("A", MessageBodySize) + +func RandomMsg(b *testing.B) (module.MsgMetadata, textproto.Header, buffer.Buffer) { + IDRaw := sha1.Sum([]byte(b.Name())) + encodedID := hex.EncodeToString(IDRaw[:]) + + body := bufio.NewReader(strings.NewReader(testMailString)) + hdr, _ := textproto.ReadHeader(body) + for i := 0; i < ExtraMessageHeaderFields; i++ { + hdr.Add("AAAAAAAAAAAA-"+strconv.Itoa(i), strings.Repeat("A", ExtraMessageHeaderFieldSize)) + } + bodyBlob, _ := io.ReadAll(body) + + return module.MsgMetadata{ + DontTraceSender: true, + ID: encodedID, + }, hdr, buffer.MemoryBuffer{Slice: bodyBlob} +} + +func BenchDelivery(b *testing.B, target module.DeliveryTarget, sender string, recipientTemplates []string) { + meta, header, body := RandomMsg(b) + + benchCtx := context.Background() + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + delivery, err := target.Start(benchCtx, &meta, sender) + if err != nil { + b.Fatal(err) + } + + for i, rcptTemplate := range recipientTemplates { + rcpt := strings.Replace(rcptTemplate, "X", strconv.Itoa(i), -1) + + if err := delivery.AddRcpt(benchCtx, rcpt, smtp.RcptOptions{}); err != nil { + b.Fatal(err) + } + } + + if err := delivery.Body(benchCtx, header, body); err != nil { + b.Fatal(err) + } + + if err := delivery.Commit(benchCtx); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/testutils/buffer.go b/internal/testutils/buffer.go new file mode 100644 index 0000000..259eea2 --- /dev/null +++ b/internal/testutils/buffer.go @@ -0,0 +1,84 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "bufio" + "bytes" + "io" + "strings" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" +) + +func BodyFromStr(t *testing.T, literal string) (textproto.Header, buffer.MemoryBuffer) { + t.Helper() + + bufr := bufio.NewReader(strings.NewReader(literal)) + hdr, err := textproto.ReadHeader(bufr) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(bufr) + if err != nil { + t.Fatal(err) + } + + return hdr, buffer.MemoryBuffer{Slice: body} +} + +type errorReader struct { + r io.Reader + err error +} + +func (r *errorReader) Read(b []byte) (int, error) { + n, err := r.r.Read(b) + if err == io.EOF { + return n, r.err + } + return n, err +} + +type FailingBuffer struct { + Blob []byte + + OpenError error + IOError error +} + +func (fb FailingBuffer) Open() (io.ReadCloser, error) { + r := io.NopCloser(bytes.NewReader(fb.Blob)) + + if fb.IOError != nil { + return io.NopCloser(&errorReader{r, fb.IOError}), fb.OpenError + } + + return r, fb.OpenError +} + +func (fb FailingBuffer) Len() int { + return len(fb.Blob) +} + +func (fb FailingBuffer) Remove() error { + return nil +} diff --git a/internal/testutils/check.go b/internal/testutils/check.go new file mode 100644 index 0000000..399a78d --- /dev/null +++ b/internal/testutils/check.go @@ -0,0 +1,111 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +type Check struct { + InitErr error + EarlyErr error + ConnRes module.CheckResult + SenderRes module.CheckResult + RcptRes module.CheckResult + BodyRes module.CheckResult + + ConnCalls int + SenderCalls int + RcptCalls int + BodyCalls int + + UnclosedStates int + + InstName string +} + +func (c *Check) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + if c.InitErr != nil { + return nil, c.InitErr + } + + c.UnclosedStates++ + return &checkState{msgMeta, c}, nil +} + +func (c *Check) Init(*config.Map) error { + return nil +} + +func (c *Check) Name() string { + return "test_check" +} + +func (c *Check) InstanceName() string { + if c.InstName != "" { + return c.InstName + } + return "test_check" +} + +func (c *Check) CheckConnection(ctx context.Context, state *module.ConnState) error { + return c.EarlyErr +} + +type checkState struct { + msgMeta *module.MsgMetadata + check *Check +} + +func (cs *checkState) CheckConnection(ctx context.Context) module.CheckResult { + cs.check.ConnCalls++ + return cs.check.ConnRes +} + +func (cs *checkState) CheckSender(ctx context.Context, from string) module.CheckResult { + cs.check.SenderCalls++ + return cs.check.SenderRes +} + +func (cs *checkState) CheckRcpt(ctx context.Context, to string) module.CheckResult { + cs.check.RcptCalls++ + return cs.check.RcptRes +} + +func (cs *checkState) CheckBody(ctx context.Context, header textproto.Header, body buffer.Buffer) module.CheckResult { + cs.check.BodyCalls++ + return cs.check.BodyRes +} + +func (cs *checkState) Close() error { + cs.check.UnclosedStates-- + return nil +} + +func init() { + module.Register("test_check", func(_, _ string, _, _ []string) (module.Module, error) { + return &Check{}, nil + }) + module.RegisterInstance(&Check{}, nil) +} diff --git a/internal/testutils/filesystem.go b/internal/testutils/filesystem.go new file mode 100644 index 0000000..cdac017 --- /dev/null +++ b/internal/testutils/filesystem.go @@ -0,0 +1,34 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "os" + "testing" +) + +// Dir is a wrapper for os.MkdirTemp that +// fails the test on errors. +func Dir(t *testing.T) string { + dir, err := os.MkdirTemp("", "maddy-tests-") + if err != nil { + t.Fatalf("can't create test dir: %v", err) + } + return dir +} diff --git a/internal/testutils/logger.go b/internal/testutils/logger.go new file mode 100644 index 0000000..9fd5506 --- /dev/null +++ b/internal/testutils/logger.go @@ -0,0 +1,59 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "flag" + "os" + "strings" + "testing" + "time" + + "github.com/foxcpp/maddy/framework/log" +) + +var ( + debugLog = flag.Bool("test.debuglog", false, "(maddy) Turn on debug log messages") + directLog = flag.Bool("test.directlog", false, "(maddy) Log to stderr instead of test log") +) + +func Logger(t *testing.T, name string) log.Logger { + if *directLog { + return log.Logger{ + Out: log.WriterOutput(os.Stderr, true), + Name: name, + Debug: *debugLog, + } + } + + return log.Logger{ + Out: log.FuncOutput(func(_ time.Time, debug bool, str string) { + t.Helper() + str = strings.TrimSuffix(str, "\n") + if debug { + str = "[debug] " + str + } + t.Log(str) + }, func() error { + return nil + }), + Name: name, + Debug: *debugLog, + } +} diff --git a/internal/testutils/modifier.go b/internal/testutils/modifier.go new file mode 100644 index 0000000..a96cbe4 --- /dev/null +++ b/internal/testutils/modifier.go @@ -0,0 +1,122 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +type Modifier struct { + InstName string + + InitErr error + MailFromErr error + RcptToErr error + BodyErr error + + MailFrom map[string]string + RcptTo map[string][]string + AddHdr textproto.Header + + UnclosedStates int +} + +func (m Modifier) Init(*config.Map) error { + return nil +} + +func (m Modifier) Name() string { + return "test_modifier" +} + +func (m Modifier) InstanceName() string { + return m.InstName +} + +type modifierState struct { + m *Modifier +} + +func (m Modifier) ModStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.ModifierState, error) { + if m.InitErr != nil { + return nil, m.InitErr + } + + m.UnclosedStates++ + return modifierState{&m}, nil +} + +func (ms modifierState) RewriteSender(ctx context.Context, mailFrom string) (string, error) { + if ms.m.MailFromErr != nil { + return "", ms.m.MailFromErr + } + if ms.m.MailFrom == nil { + return mailFrom, nil + } + + newMailFrom, ok := ms.m.MailFrom[mailFrom] + if ok { + return newMailFrom, nil + } + return mailFrom, nil +} + +func (ms modifierState) RewriteRcpt(ctx context.Context, rcptTo string) ([]string, error) { + if ms.m.RcptToErr != nil { + return []string{""}, ms.m.RcptToErr + } + + if ms.m.RcptTo == nil { + return []string{rcptTo}, nil + } + + newRcptTo, ok := ms.m.RcptTo[rcptTo] + if ok { + return newRcptTo, nil + } + return []string{rcptTo}, nil +} + +func (ms modifierState) RewriteBody(ctx context.Context, h *textproto.Header, body buffer.Buffer) error { + if ms.m.BodyErr != nil { + return ms.m.BodyErr + } + + for field := ms.m.AddHdr.Fields(); field.Next(); { + h.Add(field.Key(), field.Value()) + } + return nil +} + +func (ms modifierState) Close() error { + ms.m.UnclosedStates-- + return nil +} + +func init() { + module.Register("test_modifier", func(_, _ string, _, _ []string) (module.Module, error) { + return &Modifier{}, nil + }) + module.RegisterInstance(&Modifier{}, nil) +} diff --git a/internal/testutils/multitable.go b/internal/testutils/multitable.go new file mode 100644 index 0000000..9b84abe --- /dev/null +++ b/internal/testutils/multitable.go @@ -0,0 +1,35 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import "context" + +type MultiTable struct { + M map[string][]string + Err error +} + +func (m MultiTable) LookupMulti(_ context.Context, a string) ([]string, error) { + b, ok := m.M[a] + if ok { + return b, m.Err + } else { + return []string{}, m.Err + } +} diff --git a/internal/testutils/smtp_server.go b/internal/testutils/smtp_server.go new file mode 100644 index 0000000..9af52c4 --- /dev/null +++ b/internal/testutils/smtp_server.go @@ -0,0 +1,454 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net" + "reflect" + "sort" + "sync/atomic" + "testing" + "time" + + "github.com/emersion/go-sasl" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/exterrors" +) + +type SMTPMessage struct { + From string + Opts smtp.MailOptions + To []string + Data []byte + Conn *smtp.Conn + AuthUser string + AuthPass string +} + +type SMTPBackend struct { + Messages []*SMTPMessage + MailFromCounter int + SessionCounter int + SourceEndpoints map[string]struct{} + + AuthErr error + MailErr error + RcptErr map[string]error + DataErr error + LMTPDataErr []error + + ActiveSessionsCounter atomic.Int32 +} + +func (be *SMTPBackend) NewSession(conn *smtp.Conn) (smtp.Session, error) { + be.SessionCounter++ + be.ActiveSessionsCounter.Add(1) + if be.SourceEndpoints == nil { + be.SourceEndpoints = make(map[string]struct{}) + } + be.SourceEndpoints[conn.Conn().RemoteAddr().String()] = struct{}{} + return &session{ + backend: be, + conn: conn, + }, nil +} + +func (be *SMTPBackend) ConnectionCount() int { + return int(be.ActiveSessionsCounter.Load()) +} + +func (be *SMTPBackend) CheckMsg(t *testing.T, indx int, from string, rcptTo []string) { + t.Helper() + + if len(be.Messages) <= indx { + t.Errorf("Expected at least %d messages in mailbox, got %d", indx+1, len(be.Messages)) + return + } + + msg := be.Messages[indx] + if msg.From != from { + t.Errorf("Wrong MAIL FROM: %v", msg.From) + } + + sort.Strings(msg.To) + sort.Strings(rcptTo) + + if !reflect.DeepEqual(msg.To, rcptTo) { + t.Errorf("Wrong RCPT TO: %v", msg.To) + } + if string(msg.Data) != DeliveryData { + t.Errorf("Wrong DATA payload: %v (%v)", string(msg.Data), msg.Data) + } +} + +type session struct { + backend *SMTPBackend + user string + password string + conn *smtp.Conn + msg *SMTPMessage +} + +func (s *session) AuthMechanisms() []string { + return []string{sasl.Plain} +} + +func (s *session) Auth(mech string) (sasl.Server, error) { + if mech != sasl.Plain { + return nil, fmt.Errorf("mechanisms other than plain are unsupported") + } + return sasl.NewPlainServer(func(identity, username, password string) error { + if s.backend.AuthErr != nil { + return s.backend.AuthErr + } + s.user = username + s.password = password + return nil + }), nil +} + +func (s *session) Reset() { + s.msg = &SMTPMessage{} +} + +func (s *session) Logout() error { + s.backend.ActiveSessionsCounter.Add(-1) + return nil +} + +func (s *session) Mail(from string, opts *smtp.MailOptions) error { + s.backend.MailFromCounter++ + + if s.backend.MailErr != nil { + return s.backend.MailErr + } + + s.Reset() + s.msg.From = from + s.msg.Opts = *opts + return nil +} + +func (s *session) Rcpt(to string, _ *smtp.RcptOptions) error { + if err := s.backend.RcptErr[to]; err != nil { + return err + } + + s.msg.To = append(s.msg.To, to) + return nil +} + +func (s *session) Data(r io.Reader) error { + if s.backend.DataErr != nil { + return s.backend.DataErr + } + + b, err := io.ReadAll(r) + if err != nil { + return err + } + s.msg.Data = b + s.msg.Conn = s.conn + s.msg.AuthUser = s.user + s.msg.AuthPass = s.password + s.backend.Messages = append(s.backend.Messages, s.msg) + return nil +} + +func (s *session) LMTPData(r io.Reader, status smtp.StatusCollector) error { + if s.backend.DataErr != nil { + return s.backend.DataErr + } + + b, err := io.ReadAll(r) + if err != nil { + return err + } + s.msg.Data = b + s.msg.Conn = s.conn + s.msg.AuthUser = s.user + s.msg.AuthPass = s.password + s.backend.Messages = append(s.backend.Messages, s.msg) + + for i, rcpt := range s.msg.To { + status.SetStatus(rcpt, s.backend.LMTPDataErr[i]) + } + + return nil +} + +type SMTPServerConfigureFunc func(*smtp.Server) + +func SMTPServer(t *testing.T, addr string, fn ...SMTPServerConfigureFunc) (*SMTPBackend, *smtp.Server) { + t.Helper() + + l, err := net.Listen("tcp", addr) + if err != nil { + t.Fatal(err) + } + + be := new(SMTPBackend) + s := smtp.NewServer(be) + s.Domain = "localhost" + s.AllowInsecureAuth = true + for _, f := range fn { + f(s) + } + + go func() { + if err := s.Serve(l); err != nil { + t.Error(err) + } + }() + + // Dial it once it make sure Server completes its initialization before + // we try to use it. Notably, if test fails before connecting to the server, + // it will call Server.Close which will call Server.listener.Close with a + // nil Server.listener (Serve sets it to a non-nil value, so it is racy and + // happens only sometimes). + testConn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + testConn.Close() + + return be, s +} + +// RSA 1024, valid for *.example.invalid, 127.0.0.1, 127.0.0.2,, 127.0.0.3 +// until Nov 18 17:13:45 2029 GMT. +const testServerCert = `-----BEGIN CERTIFICATE----- +MIICDzCCAXigAwIBAgIRAJ1x+qCW7L+Hs6sRU8BHmWkwDQYJKoZIhvcNAQELBQAw +EjEQMA4GA1UEChMHQWNtZSBDbzAeFw0xOTExMTgxNzEzNDVaFw0yOTExMTUxNzEz +NDVaMBIxEDAOBgNVBAoTB0FjbWUgQ28wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ +AoGBAPINKMyuu3AvzndLDS2/BroA+DRUcAhWPBxMxG1b1BkkHisAZWteKajKmwdO +O13N8HHBRPPOD56AAPLZGNxYLHn6nel7AiH8k40/xC5tDOthqA82+00fwJHDFCnW +oDLOLcO17HulPvfCSWfefc+uee4kajPa+47hutqZH2bGMTXhAgMBAAGjZTBjMA4G +A1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAA +MC4GA1UdEQQnMCWCESouZXhhbXBsZS5pbnZhbGlkhwR/AAABhwR/AAAChwR/AAAD +MA0GCSqGSIb3DQEBCwUAA4GBAGRn3C2NbwR4cyQmTRm5jcaqi1kAYyEu6U8Q9PJW +Q15BXMKUTx2lw//QScK9MH2JpKxDuzWDSvaxZMnTxgri2uiplqpe8ydsWj6Wl0q9 +2XMGJ9LIxTZk5+cyZP2uOolvmSP/q8VFTyk9Udl6KUZPQyoiiDq4rBFUIxUyb+bX +pHkR +-----END CERTIFICATE-----` + +const testServerKey = `-----BEGIN PRIVATE KEY----- +MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAPINKMyuu3AvzndL +DS2/BroA+DRUcAhWPBxMxG1b1BkkHisAZWteKajKmwdOO13N8HHBRPPOD56AAPLZ +GNxYLHn6nel7AiH8k40/xC5tDOthqA82+00fwJHDFCnWoDLOLcO17HulPvfCSWfe +fc+uee4kajPa+47hutqZH2bGMTXhAgMBAAECgYEAgPjSDH3uEdDnSlkLJJzskJ+D +oR58s3R/gvTElSCg2uSLzo3ffF4oBHAwOqxMpabdvz8j5mSdne7Gkp9qx72TtEG2 +wt6uX1tZhm2UTAkInH8IQDthj98P8vAWQsS6HHEIMErsrW2CyUrAt/+o1BRg/hWW +zixA3CLTthhZTJkaUCECQQD5EM16UcTAKfhr3IZppgq+ZsAOMkeCl3XVV9gHo32i +DL6UFAb27BAYyjfcZB1fPou4RszX0Ryu9yU0P5qm6N47AkEA+MpdAPkaPziY0ok4 +e9Tcee6P0mIR+/AHk9GliVX2P74DDoOHyMXOSRBwdb+z2tYjrdjkNEL1Txe+sHny +k/EukwJBAOBqlmqPwNNRPeiaRHZvSSD0XjqsbSirJl48D4gadPoNt66fOQNGAt8D +Xj/z6U9HgQdiq/IOFmVEhT5FzSh1jL8CQQD3Myth8iGQO84tM0c6U3CWfuHMqsEv +0XnV+HNAmHdLMqOa4joi1dh4ZKs5dDdi828UJ/PnsbhI1FEWzLSpJvWdAkAkVWqf +AC/TvWvEZLA6Z5CllyNzZJ7XvtIaNOosxHDolyZ1HMWMlfEb2K2ZXWLy5foKPeoY +Xi3olS9rB0J+Rvjz +-----END PRIVATE KEY-----` + +// SMTPServerSTARTTLS starts a server listening on the specified addr with the +// STARTTLS extension supported. +// +// Returned *tls.Config is for the client and is set to trust the server +// certificate. +func SMTPServerSTARTTLS(t *testing.T, addr string, fn ...SMTPServerConfigureFunc) (*tls.Config, *SMTPBackend, *smtp.Server) { + t.Helper() + + cert, err := tls.X509KeyPair([]byte(testServerCert), []byte(testServerKey)) + if err != nil { + panic(err) + } + + l, err := net.Listen("tcp", addr) + if err != nil { + t.Fatal(err) + } + + be := new(SMTPBackend) + s := smtp.NewServer(be) + s.Domain = "localhost" + s.AllowInsecureAuth = true + s.TLSConfig = &tls.Config{ + Certificates: []tls.Certificate{cert}, + } + for _, f := range fn { + f(s) + } + + pool := x509.NewCertPool() + pool.AppendCertsFromPEM([]byte(testServerCert)) + + clientCfg := &tls.Config{ + ServerName: "127.0.0.1", + Time: func() time.Time { + return time.Date(2019, time.November, 18, 17, 59, 41, 0, time.UTC) + }, + RootCAs: pool, + } + + go func() { + if err := s.Serve(l); err != nil { + t.Error(err) + } + }() + + // Dial it once it make sure Server completes its initialization before + // we try to use it. Notably, if test fails before connecting to the server, + // it will call Server.Close which will call Server.listener.Close with a + // nil Server.listener (Serve sets it to a non-nil value, so it is racy and + // happens only sometimes). + testConn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + testConn.Close() + + return clientCfg, be, s +} + +// SMTPServerTLS starts a SMTP server listening on the specified addr with +// Implicit TLS. +func SMTPServerTLS(t *testing.T, addr string, fn ...SMTPServerConfigureFunc) (*tls.Config, *SMTPBackend, *smtp.Server) { + t.Helper() + + cert, err := tls.X509KeyPair([]byte(testServerCert), []byte(testServerKey)) + if err != nil { + panic(err) + } + + l, err := tls.Listen("tcp", addr, &tls.Config{ + Certificates: []tls.Certificate{cert}, + }) + if err != nil { + t.Fatal(err) + } + + be := new(SMTPBackend) + s := smtp.NewServer(be) + s.Domain = "localhost" + for _, f := range fn { + f(s) + } + + pool := x509.NewCertPool() + pool.AppendCertsFromPEM([]byte(testServerCert)) + + clientCfg := &tls.Config{ + ServerName: "127.0.0.1", + Time: func() time.Time { + return time.Date(2019, time.November, 18, 17, 59, 41, 0, time.UTC) + }, + RootCAs: pool, + } + + go func() { + if err := s.Serve(l); err != nil { + t.Error(err) + } + }() + + // Dial it once it make sure Server completes its initialization before + // we try to use it. Notably, if test fails before connecting to the server, + // it will call Server.Close which will call Server.listener.Close with a + // nil Server.listener (Serve sets it to a non-nil value, so it is racy and + // happens only sometimes). + testConn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + testConn.Close() + + return clientCfg, be, s +} + +type smtpBackendConnCounter interface { + ConnectionCount() int +} + +func CheckSMTPConnLeak(t *testing.T, srv *smtp.Server) { + t.Helper() + + ccb, ok := srv.Backend.(smtpBackendConnCounter) + if !ok { + t.Error("CheckSMTPConnLeak used for smtp.Server with backend without ConnectionCount method") + return + } + + // Connection closure is handled asynchronously, so before failing + // wait a bit for handleQuit in go-smtp to do its work. + for i := 0; i < 10; i++ { + if ccb.ConnectionCount() == 0 { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Error("Non-closed connections present after test completion") +} + +func WaitForConnsClose(t *testing.T, srv *smtp.Server) { + t.Helper() + CheckSMTPConnLeak(t, srv) +} + +// FailOnConn fails the test if attempt is made to connect the +// specified endpoint. +func FailOnConn(t *testing.T, addr string) net.Listener { + t.Helper() + + tarpit, err := net.Listen("tcp", addr) + if err != nil { + t.Fatal(err) + } + go func() { + t.Helper() + + _, err := tarpit.Accept() + if err == nil { + t.Error("No connection expected") + } + }() + return tarpit +} + +func CheckSMTPErr(t *testing.T, err error, code int, enchCode exterrors.EnhancedCode, msg string) { + t.Helper() + + if err == nil { + t.Error("Expected an error, got none") + return + } + + fields := exterrors.Fields(err) + if val, _ := fields["smtp_code"].(int); val != code { + t.Errorf("Wrong smtp_code: %v", val) + } + if val, _ := fields["smtp_enchcode"].(exterrors.EnhancedCode); val != enchCode { + t.Errorf("Wrong smtp_enchcode: %v", val) + } + if val, _ := fields["smtp_msg"].(string); val != msg { + t.Errorf("Wrong smtp_msg: %v", val) + } +} diff --git a/internal/testutils/table.go b/internal/testutils/table.go new file mode 100644 index 0000000..ec108a8 --- /dev/null +++ b/internal/testutils/table.go @@ -0,0 +1,31 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import "context" + +type Table struct { + M map[string]string + Err error +} + +func (m Table) Lookup(_ context.Context, a string) (string, bool, error) { + b, ok := m.M[a] + return b, ok, m.Err +} diff --git a/internal/testutils/target.go b/internal/testutils/target.go new file mode 100644 index 0000000..68f5b39 --- /dev/null +++ b/internal/testutils/target.go @@ -0,0 +1,344 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "io" + "reflect" + "sort" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" +) + +type Msg struct { + MsgMeta *module.MsgMetadata + MailFrom string + RcptTo []string + Body []byte + Header textproto.Header +} + +type Target struct { + Messages []Msg + DiscardMessages bool + + StartErr error + RcptErr map[string]error + BodyErr error + PartialBodyErr map[string]error + AbortErr error + CommitErr error + + InstName string +} + +/* +module.Module is implemented with dummy functions for logging done by MsgPipeline code. +*/ + +func (dt Target) Init(*config.Map) error { + return nil +} + +func (dt Target) InstanceName() string { + if dt.InstName != "" { + return dt.InstName + } + return "test_instance" +} + +func (dt Target) Name() string { + return "test_target" +} + +type testTargetDelivery struct { + msg Msg + tgt *Target +} + +type testTargetDeliveryPartial struct { + testTargetDelivery +} + +func (dt *Target) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + if dt.PartialBodyErr != nil { + return &testTargetDeliveryPartial{ + testTargetDelivery: testTargetDelivery{ + tgt: dt, + msg: Msg{MsgMeta: msgMeta, MailFrom: mailFrom}, + }, + }, dt.StartErr + } + return &testTargetDelivery{ + tgt: dt, + msg: Msg{MsgMeta: msgMeta, MailFrom: mailFrom}, + }, dt.StartErr +} + +func (dtd *testTargetDelivery) AddRcpt(ctx context.Context, to string, _ smtp.RcptOptions) error { + if dtd.tgt.RcptErr != nil { + if err := dtd.tgt.RcptErr[to]; err != nil { + return err + } + } + + dtd.msg.RcptTo = append(dtd.msg.RcptTo, to) + return nil +} + +func (dtd *testTargetDeliveryPartial) BodyNonAtomic(ctx context.Context, c module.StatusCollector, header textproto.Header, buf buffer.Buffer) { + if dtd.tgt.PartialBodyErr != nil { + for rcpt, err := range dtd.tgt.PartialBodyErr { + c.SetStatus(rcpt, err) + } + return + } + + dtd.msg.Header = header + + body, err := buf.Open() + if err != nil { + for rcpt, err := range dtd.tgt.PartialBodyErr { + c.SetStatus(rcpt, err) + } + return + } + defer body.Close() + + dtd.msg.Body, err = io.ReadAll(body) + if err != nil { + for rcpt, err := range dtd.tgt.PartialBodyErr { + c.SetStatus(rcpt, err) + } + } +} + +func (dtd *testTargetDelivery) Body(ctx context.Context, header textproto.Header, buf buffer.Buffer) error { + if dtd.tgt.PartialBodyErr != nil { + return errors.New("partial failure occurred, no additional information available") + } + if dtd.tgt.BodyErr != nil { + return dtd.tgt.BodyErr + } + + dtd.msg.Header = header + + body, err := buf.Open() + if err != nil { + return err + } + defer body.Close() + + if dtd.tgt.DiscardMessages { + // Don't bother. + _, err = io.Copy(io.Discard, body) + return err + } + + dtd.msg.Body, err = io.ReadAll(body) + return err +} + +func (dtd *testTargetDelivery) Abort(ctx context.Context) error { + return dtd.tgt.AbortErr +} + +func (dtd *testTargetDelivery) Commit(ctx context.Context) error { + if dtd.tgt.CommitErr != nil { + return dtd.tgt.CommitErr + } + if dtd.tgt.DiscardMessages { + return nil + } + dtd.tgt.Messages = append(dtd.tgt.Messages, dtd.msg) + return nil +} + +func DoTestDelivery(t *testing.T, tgt module.DeliveryTarget, from string, to []string) string { + t.Helper() + return DoTestDeliveryMeta(t, tgt, from, to, &module.MsgMetadata{ + OriginalFrom: from, + }) +} + +func DoTestDeliveryMeta(t *testing.T, tgt module.DeliveryTarget, from string, to []string, msgMeta *module.MsgMetadata) string { + t.Helper() + + id, err := DoTestDeliveryErrMeta(t, tgt, from, to, msgMeta) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + return id +} + +func DoTestDeliveryNonAtomic(t *testing.T, c module.StatusCollector, tgt module.DeliveryTarget, from string, to []string) string { + t.Helper() + + IDRaw := sha1.Sum([]byte(t.Name())) + encodedID := hex.EncodeToString(IDRaw[:]) + + testCtx := context.Background() + + body := buffer.MemoryBuffer{Slice: []byte("foobar\r\n")} + msgMeta := module.MsgMetadata{ + DontTraceSender: true, + ID: encodedID, + OriginalFrom: from, + } + t.Log("-- tgt.Start", from) + delivery, err := tgt.Start(testCtx, &msgMeta, from) + if err != nil { + t.Log("-- ... tgt.Start", from, err, exterrors.Fields(err)) + t.Fatalf("Unexpected err: %v %+v", err, exterrors.Fields(err)) + return encodedID + } + for _, rcpt := range to { + t.Log("-- delivery.AddRcpt", rcpt) + if err := delivery.AddRcpt(testCtx, rcpt, smtp.RcptOptions{}); err != nil { + t.Log("-- ... delivery.AddRcpt", rcpt, err, exterrors.Fields(err)) + t.Log("-- delivery.Abort") + if err := delivery.Abort(testCtx); err != nil { + t.Log("-- delivery.Abort:", err, exterrors.Fields(err)) + } + t.Fatalf("Unexpected err: %v %+v", err, exterrors.Fields(err)) + return encodedID + } + } + t.Log("-- delivery.BodyNonAtomic") + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + delivery.(module.PartialDelivery).BodyNonAtomic(testCtx, c, hdr, body) + t.Log("-- delivery.Commit") + if err := delivery.Commit(testCtx); err != nil { + t.Fatalf("Unexpected err: %v %+v", err, exterrors.Fields(err)) + } + + return encodedID +} + +const DeliveryData = "A: 1\r\n" + + "B: 2\r\n" + + "\r\n" + + "foobar\r\n" + +func DoTestDeliveryErr(t *testing.T, tgt module.DeliveryTarget, from string, to []string) (string, error) { + return DoTestDeliveryErrMeta(t, tgt, from, to, &module.MsgMetadata{}) +} + +func DoTestDeliveryErrMeta(t *testing.T, tgt module.DeliveryTarget, from string, to []string, msgMeta *module.MsgMetadata) (string, error) { + t.Helper() + + IDRaw := sha1.Sum([]byte(t.Name())) + encodedID := hex.EncodeToString(IDRaw[:]) + testCtx := context.Background() + + body := buffer.MemoryBuffer{Slice: []byte("foobar\r\n")} + msgMeta.DontTraceSender = true + msgMeta.ID = encodedID + t.Log("-- tgt.Start", from) + delivery, err := tgt.Start(testCtx, msgMeta, from) + if err != nil { + t.Log("-- ... tgt.Start", from, err, exterrors.Fields(err)) + return encodedID, err + } + for _, rcpt := range to { + t.Log("-- delivery.AddRcpt", rcpt) + if err := delivery.AddRcpt(testCtx, rcpt, smtp.RcptOptions{}); err != nil { + t.Log("-- ... delivery.AddRcpt", rcpt, err, exterrors.Fields(err)) + t.Log("-- delivery.Abort") + if err := delivery.Abort(testCtx); err != nil { + t.Log("-- delivery.Abort:", err, exterrors.Fields(err)) + } + return encodedID, err + } + } + t.Log("-- delivery.Body") + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + if err := delivery.Body(testCtx, hdr, body); err != nil { + t.Log("-- ... delivery.Body", err, exterrors.Fields(err)) + t.Log("-- delivery.Abort") + if err := delivery.Abort(testCtx); err != nil { + t.Log("-- ... delivery.Abort:", err, exterrors.Fields(err)) + } + return encodedID, err + } + t.Log("-- delivery.Commit") + if err := delivery.Commit(testCtx); err != nil { + t.Log("-- ... delivery.Commit", err, exterrors.Fields(err)) + return encodedID, err + } + + return encodedID, err +} + +func CheckTestMessage(t *testing.T, tgt *Target, indx int, sender string, rcpt []string) { + t.Helper() + + if len(tgt.Messages) <= indx { + t.Errorf("wrong amount of messages received, want at least %d, got %d", indx+1, len(tgt.Messages)) + return + } + msg := tgt.Messages[indx] + + CheckMsg(t, &msg, sender, rcpt) +} + +func CheckMsg(t *testing.T, msg *Msg, sender string, rcpt []string) { + t.Helper() + + idRaw := sha1.Sum([]byte(t.Name())) + encodedId := hex.EncodeToString(idRaw[:]) + + CheckMsgID(t, msg, sender, rcpt, encodedId) +} + +func CheckMsgID(t *testing.T, msg *Msg, sender string, rcpt []string, id string) string { + t.Helper() + + if msg.MsgMeta.ID != id && id != "" { + t.Errorf("empty or wrong delivery context for passed message? %+v", msg.MsgMeta) + } + if msg.MailFrom != sender { + t.Errorf("wrong sender, want %s, got %s", sender, msg.MailFrom) + } + + sort.Strings(rcpt) + sort.Strings(msg.RcptTo) + if !reflect.DeepEqual(msg.RcptTo, rcpt) { + t.Errorf("wrong recipients, want %v, got %v", rcpt, msg.RcptTo) + } + if string(msg.Body) != "foobar\r\n" { + t.Errorf("wrong body, want '%s', got '%s' (%v)", "foobar\r\n", string(msg.Body), msg.Body) + } + + return msg.MsgMeta.ID +} diff --git a/internal/tls/acme/acme.go b/internal/tls/acme/acme.go new file mode 100644 index 0000000..a09c3e0 --- /dev/null +++ b/internal/tls/acme/acme.go @@ -0,0 +1,164 @@ +package acme + +import ( + "context" + "crypto/tls" + "fmt" + "path/filepath" + + "github.com/caddyserver/certmagic" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +const modName = "tls.loader.acme" + +type Loader struct { + instName string + + store certmagic.Storage + cache *certmagic.Cache + cfg *certmagic.Config + cancelManage context.CancelFunc + + log log.Logger +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, fmt.Errorf("%s: no inline args expected", modName) + } + return &Loader{ + instName: instName, + log: log.Logger{Name: modName}, + }, nil +} + +func (l *Loader) Init(cfg *config.Map) error { + var ( + hostname string + extraNames []string + storePath string + caPath string + testCAPath string + email string + agreed bool + challenge string + overrideDomain string + provider certmagic.DNSProvider + ) + cfg.Bool("debug", true, false, &l.log.Debug) + cfg.String("hostname", true, true, "", &hostname) + cfg.StringList("extra_names", false, false, nil, &extraNames) + cfg.String("store_path", false, false, + filepath.Join(config.StateDirectory, "acme"), &storePath) + cfg.String("ca", false, false, + certmagic.LetsEncryptProductionCA, &caPath) + cfg.String("test_ca", false, false, + certmagic.LetsEncryptStagingCA, &testCAPath) + cfg.String("email", false, false, + "", &email) + cfg.String("override_domain", false, false, + "", &overrideDomain) + cfg.Bool("agreed", false, false, &agreed) + cfg.Enum("challenge", false, true, + []string{"dns-01"}, "dns-01", &challenge) + cfg.Custom("dns", false, false, func() (interface{}, error) { + return nil, nil + }, func(m *config.Map, node config.Node) (interface{}, error) { + var p certmagic.DNSProvider + err := modconfig.ModuleFromNode("libdns", node.Args, node, m.Globals, &p) + return p, err + }, &provider) + if _, err := cfg.Process(); err != nil { + return err + } + + cmLog := l.log.Zap() + + l.store = &certmagic.FileStorage{Path: storePath} + l.cache = certmagic.NewCache(certmagic.CacheOptions{ + Logger: cmLog, + GetConfigForCert: func(c certmagic.Certificate) (*certmagic.Config, error) { + return l.cfg, nil + }, + }) + + l.cfg = certmagic.New(l.cache, certmagic.Config{ + Storage: l.store, // not sure if it is necessary to set these twice + Logger: cmLog, + DefaultServerName: hostname, + }) + issuer := certmagic.NewACMEIssuer(l.cfg, certmagic.ACMEIssuer{ + Logger: cmLog, + CA: caPath, + TestCA: testCAPath, + Email: email, + Agreed: agreed, + }) + + switch challenge { + case "dns-01": + issuer.DisableTLSALPNChallenge = true + issuer.DisableHTTPChallenge = true + if provider == nil { + return fmt.Errorf("tls.loader.acme: dns-01 challenge requires a configured DNS provider") + } + issuer.DNS01Solver = &certmagic.DNS01Solver{ + DNSManager: certmagic.DNSManager{ + DNSProvider: provider, + OverrideDomain: overrideDomain, + }, + } + default: + return fmt.Errorf("tls.loader.acme: challenge not supported") + } + l.cfg.Issuers = []certmagic.Issuer{issuer} + + if module.NoRun { + return nil + } + + manageCtx, cancelManage := context.WithCancel(context.Background()) + err := l.cfg.ManageAsync(manageCtx, append([]string{hostname}, extraNames...)) + if err != nil { + cancelManage() + return err + } + l.cancelManage = cancelManage + + return nil +} + +func (l *Loader) ConfigureTLS(c *tls.Config) error { + c.GetCertificate = l.cfg.GetCertificate + return nil +} + +func (l *Loader) Close() error { + l.cancelManage() + l.cache.Stop() + return nil +} + +func (l *Loader) Name() string { + return modName +} + +func (l *Loader) InstanceName() string { + return l.instName +} + +func init() { + hooks.AddHook(hooks.EventShutdown, func() { + certmagic.CleanUpOwnLocks(context.TODO(), log.DefaultLogger.Zap()) + }) +} + +func init() { + var _ module.TLSLoader = &Loader{} + module.Register(modName, New) +} diff --git a/internal/tls/file.go b/internal/tls/file.go new file mode 100644 index 0000000..943a59e --- /dev/null +++ b/internal/tls/file.go @@ -0,0 +1,168 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tls + +import ( + "crypto/tls" + "errors" + "fmt" + "path/filepath" + "sync" + "time" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +type FileLoader struct { + instName string + inlineArgs []string + certPaths []string + keyPaths []string + log log.Logger + + certs []tls.Certificate + certsLock sync.RWMutex + + reloadTick *time.Ticker + stopTick chan struct{} +} + +func NewFileLoader(_, instName string, _, inlineArgs []string) (module.Module, error) { + return &FileLoader{ + instName: instName, + inlineArgs: inlineArgs, + log: log.Logger{Name: "tls.loader.file", Debug: log.DefaultLogger.Debug}, + stopTick: make(chan struct{}), + }, nil +} + +func (f *FileLoader) Init(cfg *config.Map) error { + cfg.StringList("certs", false, false, nil, &f.certPaths) + cfg.StringList("keys", false, false, nil, &f.keyPaths) + if _, err := cfg.Process(); err != nil { + return err + } + + if len(f.certPaths) != len(f.keyPaths) { + return errors.New("tls.loader.file: mismatch in certs and keys count") + } + + if len(f.inlineArgs)%2 != 0 { + return errors.New("tls.loader.file: odd amount of arguments") + } + for i := 0; i < len(f.inlineArgs); i += 2 { + f.certPaths = append(f.certPaths, f.inlineArgs[i]) + f.keyPaths = append(f.keyPaths, f.inlineArgs[i+1]) + } + + for _, certPath := range f.certPaths { + if !filepath.IsAbs(certPath) { + return fmt.Errorf("tls.loader.file: only absolute paths allowed in certificate paths: sorry :(") + } + } + + if err := f.loadCerts(); err != nil { + return err + } + + hooks.AddHook(hooks.EventReload, func() { + f.log.Println("reloading certificates") + if err := f.loadCerts(); err != nil { + f.log.Error("reload failed", err) + } + }) + + f.reloadTick = time.NewTicker(time.Minute) + go f.reloadTicker() + return nil +} + +func (f *FileLoader) Close() error { + f.reloadTick.Stop() + f.stopTick <- struct{}{} + return nil +} + +func (f *FileLoader) Name() string { + return "tls.loader.file" +} + +func (f *FileLoader) InstanceName() string { + return f.instName +} + +func (f *FileLoader) reloadTicker() { + for { + select { + case <-f.reloadTick.C: + f.log.Debugln("reloading certs") + if err := f.loadCerts(); err != nil { + f.log.Error("reload failed", err) + } + case <-f.stopTick: + return + } + } +} + +func (f *FileLoader) loadCerts() error { + if len(f.certPaths) != len(f.keyPaths) { + return errors.New("mismatch in certs and keys count") + } + + if len(f.certPaths) == 0 { + return errors.New("tls.loader.file: at least one certificate required") + } + + certs := make([]tls.Certificate, 0, len(f.certPaths)) + + for i := range f.certPaths { + certPath := f.certPaths[i] + keyPath := f.keyPaths[i] + + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return fmt.Errorf("failed to load %s and %s: %v", certPath, keyPath, err) + } + certs = append(certs, cert) + } + + f.certsLock.Lock() + defer f.certsLock.Unlock() + f.certs = certs + + return nil +} + +func (f *FileLoader) ConfigureTLS(c *tls.Config) error { + // Loader function replaces only the whole slice. + f.certsLock.RLock() + defer f.certsLock.RUnlock() + + c.Certificates = f.certs + return nil +} + +func init() { + var _ module.TLSLoader = &FileLoader{} + module.Register("tls.loader.file", NewFileLoader) +} diff --git a/internal/tls/self_signed.go b/internal/tls/self_signed.go new file mode 100644 index 0000000..d5e174f --- /dev/null +++ b/internal/tls/self_signed.go @@ -0,0 +1,112 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tls + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net" + "time" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +type SelfSignedLoader struct { + instName string + serverNames []string + + cert tls.Certificate +} + +func NewSelfSignedLoader(_, instName string, _, inlineArgs []string) (module.Module, error) { + return &SelfSignedLoader{ + instName: instName, + serverNames: inlineArgs, + }, nil +} + +func (f *SelfSignedLoader) Init(cfg *config.Map) error { + if _, err := cfg.Process(); err != nil { + return err + } + + privKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return err + } + + notBefore := time.Now() + notAfter := notBefore.Add(24 * time.Hour * 7) + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return err + } + cert := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{Organization: []string{"Maddy Self-Signed"}}, + NotBefore: notBefore, + NotAfter: notAfter, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + + for _, name := range f.serverNames { + if ip := net.ParseIP(name); ip != nil { + cert.IPAddresses = append(cert.IPAddresses, ip) + } else { + cert.DNSNames = append(cert.DNSNames, name) + } + } + derBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &privKey.PublicKey, privKey) + if err != nil { + return err + } + + f.cert = tls.Certificate{ + Certificate: [][]byte{derBytes}, + PrivateKey: privKey, + Leaf: cert, + } + return nil +} + +func (f *SelfSignedLoader) Name() string { + return "tls.loader.self_signed" +} + +func (f *SelfSignedLoader) InstanceName() string { + return f.instName +} + +func (f *SelfSignedLoader) ConfigureTLS(c *tls.Config) error { + c.Certificates = []tls.Certificate{f.cert} + return nil +} + +func init() { + var _ module.TLSLoader = &SelfSignedLoader{} + module.Register("tls.loader.self_signed", NewSelfSignedLoader) +} diff --git a/internal/updatepipe/backend.go b/internal/updatepipe/backend.go new file mode 100644 index 0000000..ba04df7 --- /dev/null +++ b/internal/updatepipe/backend.go @@ -0,0 +1,45 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package updatepipe + +type BackendMode int + +const ( + // ModeReplicate configures backend to both send and receive updates over + // the pipe. + ModeReplicate BackendMode = iota + + // ModePush configures backend to send updates over the pipe only. + // + // If EnableUpdatePipe(ModePush) is called for backend, its Updates() + // channel will never receive any updates. + ModePush BackendMode = iota +) + +// The Backend interface is implemented by storage backends that support both +// updates serialization using the internal updatepipe.P implementation. +// To activate this implementation, EnableUpdatePipe should be called. +type Backend interface { + // EnableUpdatePipe enables the internal update pipe implementation. + // The mode argument selects the pipe behavior. EnableUpdatePipe must be + // called before the first call to the Updates() method. + // + // This method is idempotent. All calls after a successful one do nothing. + EnableUpdatePipe(mode BackendMode) error +} diff --git a/internal/updatepipe/pubsub/pq.go b/internal/updatepipe/pubsub/pq.go new file mode 100644 index 0000000..29f9c52 --- /dev/null +++ b/internal/updatepipe/pubsub/pq.go @@ -0,0 +1,86 @@ +package pubsub + +import ( + "context" + "database/sql" + "time" + + "github.com/foxcpp/maddy/framework/log" + "github.com/lib/pq" +) + +type Msg struct { + Key string + Payload string +} + +type PqPubSub struct { + Notify chan Msg + + L *pq.Listener + sender *sql.DB + + Log log.Logger +} + +func NewPQ(dsn string) (*PqPubSub, error) { + l := &PqPubSub{ + Log: log.Logger{Name: "pgpubsub"}, + Notify: make(chan Msg), + } + l.L = pq.NewListener(dsn, 10*time.Second, time.Minute, l.eventHandler) + var err error + l.sender, err = sql.Open("postgres", dsn) + if err != nil { + return nil, err + } + + go func() { + defer close(l.Notify) + for n := range l.L.Notify { + if n == nil { + continue + } + + l.Notify <- Msg{Key: n.Channel, Payload: n.Extra} + } + }() + + return l, nil +} + +func (l *PqPubSub) Close() error { + l.sender.Close() + l.L.Close() + return nil +} + +func (l *PqPubSub) eventHandler(ev pq.ListenerEventType, err error) { + switch ev { + case pq.ListenerEventConnected: + l.Log.DebugMsg("connected") + case pq.ListenerEventReconnected: + l.Log.Msg("connection reestablished") + case pq.ListenerEventConnectionAttemptFailed: + l.Log.Error("connection attempt failed", err) + case pq.ListenerEventDisconnected: + l.Log.Msg("connection closed", "err", err) + } +} + +func (l *PqPubSub) Subscribe(_ context.Context, key string) error { + return l.L.Listen(key) +} + +func (l *PqPubSub) Unsubscribe(_ context.Context, key string) error { + return l.L.Unlisten(key) +} + +func (l *PqPubSub) Publish(key, payload string) error { + _, err := l.sender.Exec(`SELECT pg_notify($1, $2)`, key, payload) + return err +} + +func (l *PqPubSub) Listener() chan Msg { + return l.Notify +} diff --git a/internal/updatepipe/pubsub/pubsub.go b/internal/updatepipe/pubsub/pubsub.go new file mode 100644 index 0000000..64480ab --- /dev/null +++ b/internal/updatepipe/pubsub/pubsub.go @@ -0,0 +1,11 @@ +package pubsub + +import "context" + +type PubSub interface { + Subscribe(ctx context.Context, key string) error + Unsubscribe(ctx context.Context, key string) error + Publish(key, payload string) error + Listener() chan Msg + Close() error +} diff --git a/internal/updatepipe/pubsub_pipe.go b/internal/updatepipe/pubsub_pipe.go new file mode 100644 index 0000000..b1cef67 --- /dev/null +++ b/internal/updatepipe/pubsub_pipe.go @@ -0,0 +1,101 @@ +package updatepipe + +import ( + "context" + "fmt" + "os" + "strconv" + + mess "github.com/foxcpp/go-imap-mess" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/internal/updatepipe/pubsub" +) + +type PubSubPipe struct { + PubSub pubsub.PubSub + Log log.Logger +} + +func (p *PubSubPipe) Listen(upds chan<- mess.Update) error { + go func() { + for m := range p.PubSub.Listener() { + id, upd, err := parseUpdate(m.Payload) + if err != nil { + p.Log.Error("failed to parse update", err) + continue + } + if id == p.myID() { + continue + } + upds <- *upd + } + }() + return nil +} + +func (p *PubSubPipe) InitPush() error { + return nil +} + +func (p *PubSubPipe) myID() string { + return fmt.Sprintf("%d-%p", os.Getpid(), p) +} + +func (p *PubSubPipe) channel(key interface{}) (string, error) { + var psKey string + switch k := key.(type) { + case string: + psKey = k + case uint64: + psKey = "__uint64_" + strconv.FormatUint(k, 10) + default: + return "", fmt.Errorf("updatepipe: key type must be either string or uint64") + } + return psKey, nil +} + +func (p *PubSubPipe) Subscribe(key interface{}) { + psKey, err := p.channel(key) + if err != nil { + p.Log.Error("invalid key passed to Subscribe", err) + return + } + + if err := p.PubSub.Subscribe(context.TODO(), psKey); err != nil { + p.Log.Error("pubsub subscribe failed", err) + } else { + p.Log.DebugMsg("subscribed to pubsub", "channel", psKey) + } +} + +func (p *PubSubPipe) Unsubscribe(key interface{}) { + psKey, err := p.channel(key) + if err != nil { + p.Log.Error("invalid key passed to Unsubscribe", err) + return + } + + if err := p.PubSub.Unsubscribe(context.TODO(), psKey); err != nil { + p.Log.Error("pubsub unsubscribe failed", err) + } else { + p.Log.DebugMsg("unsubscribed from pubsub", "channel", psKey) + } +} + +func (p *PubSubPipe) Push(upd mess.Update) error { + psKey, err := p.channel(upd.Key) + if err != nil { + return err + } + + updBlob, err := formatUpdate(p.myID(), upd) + if err != nil { + return err + } + + return p.PubSub.Publish(psKey, updBlob) +} + +func (p *PubSubPipe) Close() error { + return p.PubSub.Close() +} diff --git a/internal/updatepipe/serialize.go b/internal/updatepipe/serialize.go new file mode 100644 index 0000000..d5a941f --- /dev/null +++ b/internal/updatepipe/serialize.go @@ -0,0 +1,69 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package updatepipe + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + mess "github.com/foxcpp/go-imap-mess" +) + +func unescapeName(s string) string { + return strings.ReplaceAll(s, "\x10", ";") +} + +func escapeName(s string) string { + return strings.ReplaceAll(s, ";", "\x10") +} + +func parseUpdate(s string) (id string, upd *mess.Update, err error) { + parts := strings.SplitN(s, ";", 2) + if len(parts) != 2 { + return "", nil, errors.New("updatepipe: mismatched parts count") + } + + upd = &mess.Update{} + dec := json.NewDecoder(strings.NewReader(unescapeName(parts[1]))) + dec.UseNumber() + err = dec.Decode(upd) + if err != nil { + return "", nil, fmt.Errorf("parseUpdate: %w", err) + } + + if val, ok := upd.Key.(json.Number); ok { + upd.Key, _ = strconv.ParseUint(val.String(), 10, 64) + } + + return parts[0], upd, nil +} + +func formatUpdate(myID string, upd mess.Update) (string, error) { + updBlob, err := json.Marshal(upd) + if err != nil { + return "", fmt.Errorf("formatUpdate: %w", err) + } + return strings.Join([]string{ + myID, + escapeName(string(updBlob)), + }, ";") + "\n", nil +} diff --git a/internal/updatepipe/unix_pipe.go b/internal/updatepipe/unix_pipe.go new file mode 100644 index 0000000..a8249f9 --- /dev/null +++ b/internal/updatepipe/unix_pipe.go @@ -0,0 +1,129 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package updatepipe + +import ( + "bufio" + "fmt" + "io" + "net" + "os" + + mess "github.com/foxcpp/go-imap-mess" + "github.com/foxcpp/maddy/framework/log" +) + +// UnixSockPipe implements the UpdatePipe interface by serializating updates +// to/from a Unix domain socket. Due to the way Unix sockets work, only one +// Listen goroutine can be running. +// +// The socket is stream-oriented and consists of the following messages: +// +// SENDER_ID;JSON_SERIALIZED_INTERNAL_OBJECT\n +// +// And SENDER_ID is Process ID and UnixSockPipe address concated as a string. +// It is used to deduplicate updates sent to Push and recevied via Listen. +// +// The SockPath field specifies the socket path to use. The actual socket +// is initialized on the first call to Listen or (Init)Push. +type UnixSockPipe struct { + SockPath string + Log log.Logger + + listener net.Listener + sender net.Conn +} + +var _ P = &UnixSockPipe{} + +func (usp *UnixSockPipe) myID() string { + return fmt.Sprintf("%d-%p", os.Getpid(), usp) +} + +func (usp *UnixSockPipe) readUpdates(conn net.Conn, updCh chan<- mess.Update) { + scnr := bufio.NewScanner(conn) + for scnr.Scan() { + id, upd, err := parseUpdate(scnr.Text()) + if err != nil { + usp.Log.Error("malformed update received", err, "str", scnr.Text()) + } + + // It is our own update, skip. + if id == usp.myID() { + continue + } + + updCh <- *upd + } +} + +func (usp *UnixSockPipe) Listen(upd chan<- mess.Update) error { + l, err := net.Listen("unix", usp.SockPath) + if err != nil { + return err + } + usp.listener = l + go func() { + for { + conn, err := l.Accept() + if err != nil { + return + } + go usp.readUpdates(conn, upd) + } + }() + return nil +} + +func (usp *UnixSockPipe) InitPush() error { + sock, err := net.Dial("unix", usp.SockPath) + if err != nil { + return err + } + + usp.sender = sock + return nil +} + +func (usp *UnixSockPipe) Push(upd mess.Update) error { + if usp.sender == nil { + if err := usp.InitPush(); err != nil { + return err + } + } + + updStr, err := formatUpdate(usp.myID(), upd) + if err != nil { + return err + } + + _, err = io.WriteString(usp.sender, updStr) + return err +} + +func (usp *UnixSockPipe) Close() error { + if usp.sender != nil { + usp.sender.Close() + } + if usp.listener != nil { + usp.listener.Close() + os.Remove(usp.SockPath) + } + return nil +} diff --git a/internal/updatepipe/update_pipe.go b/internal/updatepipe/update_pipe.go new file mode 100644 index 0000000..1427c22 --- /dev/null +++ b/internal/updatepipe/update_pipe.go @@ -0,0 +1,62 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package updatepipe implements utilities for serialization and transport of +// IMAP update objects between processes and machines. +// +// Its main goal is provide maddy command with ability to properly notify the +// server about changes without relying on it to coordinate access in the +// first place (so maddy command can work without a running server or with a +// broken server instance). +// +// Additionally, it can be used to transfer IMAP updates between replicated +// nodes. +package updatepipe + +import ( + mess "github.com/foxcpp/go-imap-mess" +) + +// The P interface represents the handle for a transport medium used for IMAP +// updates. +type P interface { + // Listen starts the "pull" goroutine that reads updates from the pipe and + // sends them to the channel. + // + // Usually it is not possible to call Listen multiple times for the same + // pipe. + // + // Updates sent using the same UpdatePipe object using Push are not + // duplicates to the channel passed to Listen. + Listen(upds chan<- mess.Update) error + + // InitPush prepares the UpdatePipe to be used as updates source (Push + // method). + // + // It is called implicitly on the first Push call, but calling it + // explicitly allows to detect initialization errors early. + InitPush() error + + // Push writes the update to the pipe. + // + // The update will not be duplicated if the UpdatePipe is also listening + // for updates. + Push(upd mess.Update) error + + Close() error +} diff --git a/maddy.conf b/maddy.conf new file mode 100644 index 0000000..5f02fb3 --- /dev/null +++ b/maddy.conf @@ -0,0 +1,181 @@ +## Maddy Mail Server - default configuration file (2022-06-18) +# Suitable for small-scale deployments. Uses its own format for local users DB, +# should be managed via maddy subcommands. +# +# See tutorials at https://maddy.email for guidance on typical +# configuration changes. + +# ---------------------------------------------------------------------------- +# Base variables + +$(hostname) = example.org +$(primary_domain) = example.org +$(local_domains) = $(primary_domain) + +tls file /etc/maddy/certs/$(hostname)/fullchain.pem /etc/maddy/certs/$(hostname)/privkey.pem + +# ---------------------------------------------------------------------------- +# Local storage & authentication + +# pass_table provides local hashed passwords storage for authentication of +# users. It can be configured to use any "table" module, in default +# configuration a table in SQLite DB is used. +# Table can be replaced to use e.g. a file for passwords. Or pass_table module +# can be replaced altogether to use some external source of credentials (e.g. +# PAM, /etc/shadow file). +# +# If table module supports it (sql_table does) - credentials can be managed +# using 'maddy creds' command. + +auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } +} + +# imapsql module stores all indexes and metadata necessary for IMAP using a +# relational database. It is used by IMAP endpoint for mailbox access and +# also by SMTP & Submission endpoints for delivery of local messages. +# +# IMAP accounts, mailboxes and all message metadata can be inspected using +# imap-* subcommands of maddy. + +storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db +} + +# ---------------------------------------------------------------------------- +# SMTP endpoints + message routing + +hostname $(hostname) + +table.chain local_rewrites { + optional_step regexp "(.+)\+(.+)@(.+)" "$1@$3" + optional_step static { + entry postmaster postmaster@$(primary_domain) + } + optional_step file /etc/maddy/aliases +} + +msgpipeline local_routing { + # Insert handling for special-purpose local domains here. + # e.g. + # destination lists.example.org { + # deliver_to lmtp tcp://127.0.0.1:8024 + # } + + destination postmaster $(local_domains) { + modify { + replace_rcpt &local_rewrites + } + + deliver_to &local_mailboxes + } + + default_destination { + reject 550 5.1.1 "User doesn't exist" + } +} + +smtp tcp://0.0.0.0:25 { + limits { + # Up to 20 msgs/sec across max. 10 SMTP connections. + all rate 20 1s + all concurrency 10 + } + + dmarc yes + check { + require_mx_record + dkim + spf + } + + source $(local_domains) { + reject 501 5.1.8 "Use Submission for outgoing SMTP" + } + default_source { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + reject 550 5.1.1 "User doesn't exist" + } + } +} + +submission tls://0.0.0.0:465 tcp://0.0.0.0:587 { + limits { + # Up to 50 msgs/sec across any amount of SMTP connections. + all rate 50 1s + } + + auth &local_authdb + + source $(local_domains) { + check { + authorize_sender { + prepare_email &local_rewrites + user_to_email identity + } + } + + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + modify { + dkim $(primary_domain) $(local_domains) default + } + deliver_to &remote_queue + } + } + default_source { + reject 501 5.1.8 "Non-local sender domain" + } +} + +target.remote outbound_delivery { + limits { + # Up to 20 msgs/sec across max. 10 SMTP connections + # for each recipient domain. + destination rate 20 1s + destination concurrency 10 + } + mx_auth { + dane + mtasts { + cache fs + fs_dir mtasts_cache/ + } + local_policy { + min_tls_level encrypted + min_mx_level none + } + } +} + +target.queue remote_queue { + target &outbound_delivery + + autogenerated_msg_domain $(primary_domain) + bounce { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + reject 550 5.0.0 "Refusing to send DSNs to non-local addresses" + } + } +} + +# ---------------------------------------------------------------------------- +# IMAP endpoints + +imap tls://0.0.0.0:993 tcp://0.0.0.0:143 { + auth &local_authdb + storage &local_mailboxes +} diff --git a/maddy.conf.docker b/maddy.conf.docker new file mode 100644 index 0000000..39c3bbb --- /dev/null +++ b/maddy.conf.docker @@ -0,0 +1,182 @@ +## Maddy Mail Server - default configuration file (2022-06-18) +## This is the copy of maddy.conf with changes necessary to run it in Docker. +# Suitable for small-scale deployments. Uses its own format for local users DB, +# should be managed via maddy subcommands. +# +# See tutorials at https://maddy.email for guidance on typical +# configuration changes. + +# ---------------------------------------------------------------------------- +# Base variables + +$(hostname) = {env:MADDY_HOSTNAME} +$(primary_domain) = {env:MADDY_DOMAIN} +$(local_domains) = $(primary_domain) + +tls file /data/tls/fullchain.pem /data/tls/privkey.pem + +# ---------------------------------------------------------------------------- +# Local storage & authentication + +# pass_table provides local hashed passwords storage for authentication of +# users. It can be configured to use any "table" module, in default +# configuration a table in SQLite DB is used. +# Table can be replaced to use e.g. a file for passwords. Or pass_table module +# can be replaced altogether to use some external source of credentials (e.g. +# PAM, /etc/shadow file). +# +# If table module supports it (sql_table does) - credentials can be managed +# using 'maddy creds' command. + +auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } +} + +# imapsql module stores all indexes and metadata necessary for IMAP using a +# relational database. It is used by IMAP endpoint for mailbox access and +# also by SMTP & Submission endpoints for delivery of local messages. +# +# IMAP accounts, mailboxes and all message metadata can be inspected using +# imap-* subcommands of maddy. + +storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db +} + +# ---------------------------------------------------------------------------- +# SMTP endpoints + message routing + +hostname $(hostname) + +table.chain local_rewrites { + optional_step regexp "(.+)\+(.+)@(.+)" "$1@$3" + optional_step static { + entry postmaster postmaster@$(primary_domain) + } + optional_step file /etc/maddy/aliases +} + +msgpipeline local_routing { + # Insert handling for special-purpose local domains here. + # e.g. + # destination lists.example.org { + # deliver_to lmtp tcp://127.0.0.1:8024 + # } + + destination postmaster $(local_domains) { + modify { + replace_rcpt &local_rewrites + } + + deliver_to &local_mailboxes + } + + default_destination { + reject 550 5.1.1 "User doesn't exist" + } +} + +smtp tcp://0.0.0.0:25 { + limits { + # Up to 20 msgs/sec across max. 10 SMTP connections. + all rate 20 1s + all concurrency 10 + } + + dmarc yes + check { + require_mx_record + dkim + spf + } + + source $(local_domains) { + reject 501 5.1.8 "Use Submission for outgoing SMTP" + } + default_source { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + reject 550 5.1.1 "User doesn't exist" + } + } +} + +submission tls://0.0.0.0:465 tcp://0.0.0.0:587 { + limits { + # Up to 50 msgs/sec across any amount of SMTP connections. + all rate 50 1s + } + + auth &local_authdb + + source $(local_domains) { + check { + authorize_sender { + prepare_email &local_rewrites + user_to_email identity + } + } + + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + modify { + dkim $(primary_domain) $(local_domains) default + } + deliver_to &remote_queue + } + } + default_source { + reject 501 5.1.8 "Non-local sender domain" + } +} + +target.remote outbound_delivery { + limits { + # Up to 20 msgs/sec across max. 10 SMTP connections + # for each recipient domain. + destination rate 20 1s + destination concurrency 10 + } + mx_auth { + dane + mtasts { + cache fs + fs_dir mtasts_cache/ + } + local_policy { + min_tls_level encrypted + min_mx_level none + } + } +} + +target.queue remote_queue { + target &outbound_delivery + + autogenerated_msg_domain $(primary_domain) + bounce { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + reject 550 5.0.0 "Refusing to send DSNs to non-local addresses" + } + } +} + +# ---------------------------------------------------------------------------- +# IMAP endpoints + +imap tls://0.0.0.0:993 tcp://0.0.0.0:143 { + auth &local_authdb + storage &local_mailboxes +} diff --git a/maddy.go b/maddy.go new file mode 100644 index 0000000..5439fba --- /dev/null +++ b/maddy.go @@ -0,0 +1,434 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package maddy + +import ( + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "runtime" + "runtime/debug" + + "github.com/caddyserver/certmagic" + parser "github.com/foxcpp/maddy/framework/cfgparser" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/authz" + maddycli "github.com/foxcpp/maddy/internal/cli" + "github.com/urfave/cli/v2" + + // Import packages for side-effect of module registration. + _ "github.com/foxcpp/maddy/internal/auth/dovecot_sasl" + _ "github.com/foxcpp/maddy/internal/auth/external" + _ "github.com/foxcpp/maddy/internal/auth/ldap" + _ "github.com/foxcpp/maddy/internal/auth/netauth" + _ "github.com/foxcpp/maddy/internal/auth/pam" + _ "github.com/foxcpp/maddy/internal/auth/pass_table" + _ "github.com/foxcpp/maddy/internal/auth/plain_separate" + _ "github.com/foxcpp/maddy/internal/auth/shadow" + _ "github.com/foxcpp/maddy/internal/check/authorize_sender" + _ "github.com/foxcpp/maddy/internal/check/command" + _ "github.com/foxcpp/maddy/internal/check/dkim" + _ "github.com/foxcpp/maddy/internal/check/dns" + _ "github.com/foxcpp/maddy/internal/check/dnsbl" + _ "github.com/foxcpp/maddy/internal/check/milter" + _ "github.com/foxcpp/maddy/internal/check/requiretls" + _ "github.com/foxcpp/maddy/internal/check/rspamd" + _ "github.com/foxcpp/maddy/internal/check/spf" + _ "github.com/foxcpp/maddy/internal/endpoint/dovecot_sasld" + _ "github.com/foxcpp/maddy/internal/endpoint/imap" + _ "github.com/foxcpp/maddy/internal/endpoint/openmetrics" + _ "github.com/foxcpp/maddy/internal/endpoint/smtp" + _ "github.com/foxcpp/maddy/internal/imap_filter" + _ "github.com/foxcpp/maddy/internal/imap_filter/command" + _ "github.com/foxcpp/maddy/internal/libdns" + _ "github.com/foxcpp/maddy/internal/modify" + _ "github.com/foxcpp/maddy/internal/modify/dkim" + _ "github.com/foxcpp/maddy/internal/storage/blob/fs" + _ "github.com/foxcpp/maddy/internal/storage/blob/s3" + _ "github.com/foxcpp/maddy/internal/storage/imapsql" + _ "github.com/foxcpp/maddy/internal/table" + _ "github.com/foxcpp/maddy/internal/target/queue" + _ "github.com/foxcpp/maddy/internal/target/remote" + _ "github.com/foxcpp/maddy/internal/target/smtp" + _ "github.com/foxcpp/maddy/internal/tls" + _ "github.com/foxcpp/maddy/internal/tls/acme" +) + +var ( + Version = "go-build" + + enableDebugFlags = false +) + +func BuildInfo() string { + version := Version + if info, ok := debug.ReadBuildInfo(); ok && info.Main.Version != "(devel)" { + version = info.Main.Version + } + + return fmt.Sprintf(`%s %s/%s %s + +default config: %s +default state_dir: %s +default runtime_dir: %s`, + version, runtime.GOOS, runtime.GOARCH, runtime.Version(), + filepath.Join(ConfigDirectory, "maddy.conf"), + DefaultStateDirectory, + DefaultRuntimeDirectory) +} + +func init() { + maddycli.AddGlobalFlag( + &cli.PathFlag{ + Name: "config", + Usage: "Configuration file to use", + EnvVars: []string{"MADDY_CONFIG"}, + Value: filepath.Join(ConfigDirectory, "maddy.conf"), + }, + ) + maddycli.AddGlobalFlag(&cli.BoolFlag{ + Name: "debug", + Usage: "enable debug logging early", + Destination: &log.DefaultLogger.Debug, + }) + maddycli.AddSubcommand(&cli.Command{ + Name: "run", + Usage: "Start the server", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "libexec", + Value: DefaultLibexecDirectory, + Usage: "path to the libexec directory", + Destination: &config.LibexecDirectory, + }, + &cli.StringSliceFlag{ + Name: "log", + Usage: "default logging target(s)", + Value: cli.NewStringSlice("stderr"), + }, + &cli.BoolFlag{ + Name: "v", + Usage: "print version and build metadata, then exit", + Hidden: true, + }, + }, + Action: Run, + }) + maddycli.AddSubcommand(&cli.Command{ + Name: "version", + Usage: "Print version and build metadata, then exit", + Action: func(c *cli.Context) error { + fmt.Println(BuildInfo()) + return nil + }, + }) + + if enableDebugFlags { + maddycli.AddGlobalFlag(&cli.StringFlag{ + Name: "debug.pprof", + Usage: "enable live profiler HTTP endpoint and listen on the specified address", + }) + maddycli.AddGlobalFlag(&cli.IntFlag{ + Name: "debug.blockprofrate", + Usage: "set blocking profile rate", + }) + maddycli.AddGlobalFlag(&cli.IntFlag{ + Name: "debug.mutexproffract", + Usage: "set mutex profile fraction", + }) + } +} + +// Run is the entry point for all server-running code. It takes care of command line arguments processing, +// logging initialization, directives setup, configuration reading. After all that, it +// calls moduleMain to initialize and run modules. +func Run(c *cli.Context) error { + certmagic.UserAgent = "maddy/" + Version + + if c.NArg() != 0 { + return cli.Exit(fmt.Sprintln("usage:", os.Args[0], "[options]"), 2) + } + + if c.Bool("v") { + fmt.Println("maddy", BuildInfo()) + return nil + } + + var err error + log.DefaultLogger.Out, err = LogOutputOption(c.StringSlice("log")) + if err != nil { + systemdStatusErr(err) + return cli.Exit(err.Error(), 2) + } + + initDebug(c) + + os.Setenv("PATH", config.LibexecDirectory+string(filepath.ListSeparator)+os.Getenv("PATH")) + + f, err := os.Open(c.Path("config")) + if err != nil { + systemdStatusErr(err) + return cli.Exit(err.Error(), 2) + } + defer f.Close() + + cfg, err := parser.Read(f, c.Path("config")) + if err != nil { + systemdStatusErr(err) + return cli.Exit(err.Error(), 2) + } + + defer log.DefaultLogger.Out.Close() + + if err := moduleMain(cfg); err != nil { + systemdStatusErr(err) + return cli.Exit(err.Error(), 1) + } + + return nil +} + +func initDebug(c *cli.Context) { + if !enableDebugFlags { + return + } + + if c.IsSet("debug.pprof") { + profileEndpoint := c.String("debug.pprof") + go func() { + log.Println("listening on", "http://"+profileEndpoint, "for profiler requests") + log.Println("failed to listen on profiler endpoint:", http.ListenAndServe(profileEndpoint, nil)) + }() + } + + // These values can also be affected by environment so set them + // only if argument is specified. + if c.IsSet("debug.mutexproffract") { + runtime.SetMutexProfileFraction(c.Int("debug.mutexproffract")) + } + if c.IsSet("debug.blockprofrate") { + runtime.SetBlockProfileRate(c.Int("debug.blockprofrate")) + } +} + +func InitDirs() error { + if config.StateDirectory == "" { + config.StateDirectory = DefaultStateDirectory + } + if config.RuntimeDirectory == "" { + config.RuntimeDirectory = DefaultRuntimeDirectory + } + if config.LibexecDirectory == "" { + config.LibexecDirectory = DefaultLibexecDirectory + } + + if err := ensureDirectoryWritable(config.StateDirectory); err != nil { + return err + } + if err := ensureDirectoryWritable(config.RuntimeDirectory); err != nil { + return err + } + + // Make sure all paths we are going to use are absolute + // before we change the working directory. + if !filepath.IsAbs(config.StateDirectory) { + return errors.New("statedir should be absolute") + } + if !filepath.IsAbs(config.RuntimeDirectory) { + return errors.New("runtimedir should be absolute") + } + if !filepath.IsAbs(config.LibexecDirectory) { + return errors.New("-libexec should be absolute") + } + + // Change the working directory to make all relative paths + // in configuration relative to state directory. + if err := os.Chdir(config.StateDirectory); err != nil { + log.Println(err) + } + + return nil +} + +func ensureDirectoryWritable(path string) error { + if err := os.MkdirAll(path, 0o700); err != nil { + return err + } + + testFile, err := os.Create(filepath.Join(path, "writeable-test")) + if err != nil { + return err + } + testFile.Close() + return os.RemoveAll(testFile.Name()) +} + +func ReadGlobals(cfg []config.Node) (map[string]interface{}, []config.Node, error) { + globals := config.NewMap(nil, config.Node{Children: cfg}) + globals.String("state_dir", false, false, DefaultStateDirectory, &config.StateDirectory) + globals.String("runtime_dir", false, false, DefaultRuntimeDirectory, &config.RuntimeDirectory) + globals.String("hostname", false, false, "", nil) + globals.String("autogenerated_msg_domain", false, false, "", nil) + globals.Custom("tls", false, false, nil, tls.TLSDirective, nil) + globals.Custom("tls_client", false, false, nil, tls.TLSClientBlock, nil) + globals.Bool("storage_perdomain", false, false, nil) + globals.Bool("auth_perdomain", false, false, nil) + globals.StringList("auth_domains", false, false, nil, nil) + globals.Custom("log", false, false, defaultLogOutput, logOutput, &log.DefaultLogger.Out) + globals.Bool("debug", false, log.DefaultLogger.Debug, &log.DefaultLogger.Debug) + config.EnumMapped(globals, "auth_map_normalize", true, false, authz.NormalizeFuncs, authz.NormalizeAuto, nil) + modconfig.Table(globals, "auth_map", true, false, nil, nil) + globals.AllowUnknown() + unknown, err := globals.Process() + return globals.Values, unknown, err +} + +func moduleMain(cfg []config.Node) error { + globals, modBlocks, err := ReadGlobals(cfg) + if err != nil { + return err + } + + if err := InitDirs(); err != nil { + return err + } + + hooks.AddHook(hooks.EventLogRotate, reinitLogging) + + endpoints, mods, err := RegisterModules(globals, modBlocks) + if err != nil { + return err + } + + err = initModules(globals, endpoints, mods) + if err != nil { + return err + } + + systemdStatus(SDReady, "Listening for incoming connections...") + + handleSignals() + + systemdStatus(SDStopping, "Waiting for running transactions to complete...") + + hooks.RunHooks(hooks.EventShutdown) + + return nil +} + +type ModInfo struct { + Instance module.Module + Cfg config.Node +} + +func RegisterModules(globals map[string]interface{}, nodes []config.Node) (endpoints, mods []ModInfo, err error) { + mods = make([]ModInfo, 0, len(nodes)) + + for _, block := range nodes { + var instName string + var modAliases []string + if len(block.Args) == 0 { + instName = block.Name + } else { + instName = block.Args[0] + modAliases = block.Args[1:] + } + + modName := block.Name + + endpFactory := module.GetEndpoint(modName) + if endpFactory != nil { + inst, err := endpFactory(modName, block.Args) + if err != nil { + return nil, nil, err + } + + endpoints = append(endpoints, ModInfo{Instance: inst, Cfg: block}) + continue + } + + factory := module.Get(modName) + if factory == nil { + return nil, nil, config.NodeErr(block, "unknown module or global directive: %s", modName) + } + + if module.HasInstance(instName) { + return nil, nil, config.NodeErr(block, "config block named %s already exists", instName) + } + + inst, err := factory(modName, instName, modAliases, nil) + if err != nil { + return nil, nil, err + } + + module.RegisterInstance(inst, config.NewMap(globals, block)) + for _, alias := range modAliases { + if module.HasInstance(alias) { + return nil, nil, config.NodeErr(block, "config block named %s already exists", alias) + } + module.RegisterAlias(alias, instName) + } + + log.Debugf("%v:%v: register config block %v %v", block.File, block.Line, instName, modAliases) + mods = append(mods, ModInfo{Instance: inst, Cfg: block}) + } + + if len(endpoints) == 0 { + return nil, nil, fmt.Errorf("at least one endpoint should be configured") + } + + return endpoints, mods, nil +} + +func initModules(globals map[string]interface{}, endpoints, mods []ModInfo) error { + for _, endp := range endpoints { + if err := endp.Instance.Init(config.NewMap(globals, endp.Cfg)); err != nil { + return err + } + + if closer, ok := endp.Instance.(io.Closer); ok { + endp := endp + hooks.AddHook(hooks.EventShutdown, func() { + log.Debugf("close %s (%s)", endp.Instance.Name(), endp.Instance.InstanceName()) + if err := closer.Close(); err != nil { + log.Printf("module %s (%s) close failed: %v", endp.Instance.Name(), endp.Instance.InstanceName(), err) + } + }) + } + } + + for _, inst := range mods { + if module.Initialized[inst.Instance.InstanceName()] { + continue + } + + return fmt.Errorf("Unused configuration block at %s:%d - %s (%s)", + inst.Cfg.File, inst.Cfg.Line, inst.Instance.InstanceName(), inst.Instance.Name()) + } + + return nil +} diff --git a/maddy_debug.go b/maddy_debug.go new file mode 100644 index 0000000..dc03168 --- /dev/null +++ b/maddy_debug.go @@ -0,0 +1,30 @@ +//go:build debugflags +// +build debugflags + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package maddy + +import ( + _ "net/http/pprof" +) + +func init() { + enableDebugFlags = true +} diff --git a/signal.go b/signal.go new file mode 100644 index 0000000..e952fcf --- /dev/null +++ b/signal.go @@ -0,0 +1,66 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package maddy + +import ( + "os" + "os/signal" + "syscall" + + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/log" +) + +// handleSignals function creates and listens on OS signals channel. +// +// OS-specific signals that correspond to the program termination +// (SIGTERM, SIGHUP, SIGINT) will cause this function to return. +// +// SIGUSR1 will call reinitLogging without returning. +func handleSignals() os.Signal { + sig := make(chan os.Signal, 5) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGINT, syscall.SIGUSR1, syscall.SIGUSR2) + + for { + switch s := <-sig; s { + case syscall.SIGUSR1: + log.Printf("signal received (%s), rotating logs", s.String()) + systemdStatus(SDReloading, "Reopening logs...") + hooks.RunHooks(hooks.EventLogRotate) + systemdStatus(SDReady, "Listening for incoming connections...") + case syscall.SIGUSR2: + log.Printf("signal received (%s), reloading state", s.String()) + systemdStatus(SDReloading, "Reloading state...") + hooks.RunHooks(hooks.EventReload) + systemdStatus(SDReady, "Listening for incoming connections...") + default: + go func() { + s := handleSignals() + log.Printf("forced shutdown due to signal (%v)!", s) + os.Exit(1) + }() + + log.Printf("signal received (%v), next signal will force immediate shutdown.", s) + return s + } + } +} diff --git a/signal_nonposix.go b/signal_nonposix.go new file mode 100644 index 0000000..87ea0cd --- /dev/null +++ b/signal_nonposix.go @@ -0,0 +1,45 @@ +//go:build windows || plan9 +// +build windows plan9 + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package maddy + +import ( + "os" + "os/signal" + "syscall" + + "github.com/foxcpp/maddy/framework/log" +) + +func handleSignals() os.Signal { + sig := make(chan os.Signal, 5) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGINT) + + s := <-sig + go func() { + s := handleSignals() + log.Printf("forced shutdown due to signal (%v)!", s) + os.Exit(1) + }() + + log.Printf("signal received (%v), next signal will force immediate shutdown.", s) + return s +} diff --git a/systemd.go b/systemd.go new file mode 100644 index 0000000..649740d --- /dev/null +++ b/systemd.go @@ -0,0 +1,133 @@ +//go:build linux +// +build linux + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package maddy + +import ( + "errors" + "fmt" + "io" + "net" + "os" + "strings" + "syscall" + + "github.com/foxcpp/maddy/framework/log" +) + +type SDStatus string + +const ( + SDReady = "READY=1" + SDReloading = "RELOADING=1" + SDStopping = "STOPPING=1" +) + +var ErrNoNotifySock = errors.New("no systemd socket") + +func sdNotifySock() (*net.UnixConn, error) { + sockAddr := os.Getenv("NOTIFY_SOCKET") + if sockAddr == "" { + return nil, ErrNoNotifySock + } + if strings.HasPrefix(sockAddr, "@") { + sockAddr = "\x00" + sockAddr[1:] + } + + return net.DialUnix("unixgram", nil, &net.UnixAddr{ + Name: sockAddr, + Net: "unixgram", + }) +} + +func setScmPassCred(sock *net.UnixConn) error { + sConn, err := sock.SyscallConn() + if err != nil { + return err + } + + var sockoptErr error + if err := sConn.Control(func(fd uintptr) { + sockoptErr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_PASSCRED, 1) + }); err != nil { + return err + } + if sockoptErr != nil { + return sockoptErr + } + return nil +} + +func systemdStatus(status SDStatus, desc string) { + sock, err := sdNotifySock() + if err != nil { + if !errors.Is(err, ErrNoNotifySock) { + log.Println("systemd: failed to acquire notify socket:", err) + } + return + } + defer sock.Close() + + if err := setScmPassCred(sock); err != nil { + log.Println("systemd: failed to set SCM_PASSCRED on the socket:", err) + } + + if desc != "" { + if _, err := io.WriteString(sock, fmt.Sprintf("%s\nSTATUS=%s", status, desc)); err != nil { + log.Println("systemd: I/O error:", err) + } + log.Debugf(`systemd: %s STATUS="%s"`, status, desc) + } else { + if _, err := io.WriteString(sock, string(status)); err != nil { + log.Println("systemd: I/O error:", err) + } + log.Debugf(`systemd: %s`, status) + } +} + +func systemdStatusErr(reportedErr error) { + sock, err := sdNotifySock() + if err != nil { + if !errors.Is(err, ErrNoNotifySock) { + log.Println("systemd: failed to acquire notify socket:", err) + } + return + } + defer sock.Close() + + if err := setScmPassCred(sock); err != nil { + log.Println("systemd: failed to set SCM_PASSCRED on the socket:", err) + } + + var errno syscall.Errno + if errors.As(reportedErr, &errno) { + log.Debugf(`systemd: ERRNO=%d STATUS="%v"`, errno, reportedErr) + if _, err := io.WriteString(sock, fmt.Sprintf("ERRNO=%d\nSTATUS=%v", errno, reportedErr)); err != nil { + log.Println("systemd: I/O error:", err) + } + return + } + + if _, err := io.WriteString(sock, fmt.Sprintf("STATUS=%v\n", reportedErr)); err != nil { + log.Println("systemd: I/O error:", err) + } + log.Debugf(`systemd: STATUS="%v"`, reportedErr) +} diff --git a/systemd_nonlinux.go b/systemd_nonlinux.go new file mode 100644 index 0000000..e31cd15 --- /dev/null +++ b/systemd_nonlinux.go @@ -0,0 +1,34 @@ +//go:build !linux +// +build !linux + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package maddy + +type SDStatus string + +const ( + SDReady = "READY=1" + SDReloading = "RELOADING=1" + SDStopping = "STOPPING=1" +) + +func systemdStatus(SDStatus, string) {} + +func systemdStatusErr(error) {} diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..765a7b4 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,17 @@ +# maddy integration testing + +## Tests structure + +The test library creates a temporary state and runtime directory, starts the +server with the specified configuration file and lets you interact with it +using a couple of convenient wrappers. + +## Running + +To run tests, use `go test -tags integration` in this directory. Make sure to +have a maddy executable in the current working directory. +Use `-integration.executable` if the executable is named different or is placed +somewhere else. +Use `-integration.coverprofile` to pass `-test.coverprofile +your_value.RANDOM` to test executable. See `./build_cover.sh` to build a +server executable instrumented with coverage counters. diff --git a/tests/basic_test.go b/tests/basic_test.go new file mode 100644 index 0000000..80cf1ae --- /dev/null +++ b/tests/basic_test.go @@ -0,0 +1,63 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "testing" + + "github.com/foxcpp/maddy/tests" +) + +func TestBasic(tt *testing.T) { + tt.Parallel() + + // This test is mostly intended to test whether the integration testing + // library is working as expected. + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.ExpectPattern("220 mx.maddy.test *") + conn.Writeln("EHLO localhost") + conn.ExpectPattern("250-*") + conn.ExpectPattern("250-PIPELINING") + conn.ExpectPattern("250-8BITMIME") + conn.ExpectPattern("250-ENHANCEDSTATUSCODES") + conn.ExpectPattern("250-CHUNKING") + conn.ExpectPattern("250-SMTPUTF8") + conn.ExpectPattern("250-SIZE *") + conn.ExpectPattern("250 LIMITS RCPTMAX=20000") + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} diff --git a/tests/build_cover.sh b/tests/build_cover.sh new file mode 100755 index 0000000..929511c --- /dev/null +++ b/tests/build_cover.sh @@ -0,0 +1,5 @@ +#!/bin/sh +if [ -z "$GO" ]; then + GO=go +fi +exec $GO test -tags 'cover_main debugflags' -coverpkg 'github.com/foxcpp/maddy,github.com/foxcpp/maddy/pkg/...,github.com/foxcpp/maddy/internal/...' -cover -covermode atomic -c cover_test.go -o maddy.cover diff --git a/tests/conn.go b/tests/conn.go new file mode 100644 index 0000000..9ff86e8 --- /dev/null +++ b/tests/conn.go @@ -0,0 +1,289 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests + +import ( + "bufio" + "crypto/tls" + "encoding/base64" + "fmt" + "io" + "net" + "path" + "strconv" + "strings" + "time" +) + +// Conn is a helper that simplifies testing of text protocol interactions. +type Conn struct { + T *T + + WriteTimeout time.Duration + ReadTimeout time.Duration + + allowIOErr bool + + Conn net.Conn + Scanner *bufio.Scanner +} + +// AllowIOErr toggles whether I/O errors should be returned to the caller of +// Conn method or should immedately fail the test. +// +// By default (ok = false), the latter happens. +func (c *Conn) AllowIOErr(ok bool) { + c.allowIOErr = ok +} + +// Write writes the string to the connection socket. +func (c *Conn) Write(s string) { + c.T.Helper() + + // Make sure the test will not accidentally hang waiting for I/O forever if + // the server breaks. + if err := c.Conn.SetWriteDeadline(time.Now().Add(c.WriteTimeout)); err != nil { + c.fatal("Cannot set write deadline: %v", err) + } + defer func() { + if err := c.Conn.SetWriteDeadline(time.Time{}); err != nil { + c.log('-', "Failed to reset connection deadline: %v", err) + } + }() + + c.log('>', "%s", s) + if _, err := io.WriteString(c.Conn, s); err != nil { + c.fatal("Unexpected I/O error: %v", err) + } +} + +func (c *Conn) Writeln(s string) { + c.T.Helper() + + c.Write(s + "\r\n") +} + +func (c *Conn) Readln() (string, error) { + c.T.Helper() + + // Make sure the test will not accidentally hang waiting for I/O forever if + // the server breaks. + if err := c.Conn.SetReadDeadline(time.Now().Add(c.ReadTimeout)); err != nil { + c.fatal("Cannot set write deadline: %v", err) + } + defer func() { + if err := c.Conn.SetReadDeadline(time.Time{}); err != nil { + c.log('-', "Failed to reset connection deadline: %v", err) + } + }() + + if !c.Scanner.Scan() { + if err := c.Scanner.Err(); err != nil { + if c.allowIOErr { + return "", err + } + c.fatal("Unexpected I/O error: %v", err) + } + if c.allowIOErr { + return "", io.EOF + } + c.fatal("Unexpected EOF") + } + + c.log('<', "%v", c.Scanner.Text()) + + return c.Scanner.Text(), nil +} + +func (c *Conn) Expect(line string) { + c.T.Helper() + + actual, err := c.Readln() + if err != nil { + c.T.Fatal("Unexpected I/O error:", err) + } + + if line != actual { + c.T.Fatalf("Response line not matching the expected one, want %q", line) + } +} + +// ExpectPattern reads a line from the connection socket and checks whether is +// matches the supplied shell pattern (as defined by path.Match). The original +// line is returned. +func (c *Conn) ExpectPattern(pat string) string { + c.T.Helper() + + line, err := c.Readln() + if err != nil { + c.T.Fatal("Unexpected I/O error:", err) + } + + match, err := path.Match(pat, line) + if err != nil { + c.T.Fatal("Malformed pattern:", err) + } + if !match { + c.T.Fatalf("Response line not matching the expected pattern, want %q", pat) + } + + return line +} + +func (c *Conn) fatal(f string, args ...interface{}) { + c.T.Helper() + c.log('-', f, args...) + c.T.FailNow() +} + +func (c *Conn) log(direction rune, f string, args ...interface{}) { + c.T.Helper() + + local, remote := c.Conn.LocalAddr().(*net.TCPAddr), c.Conn.RemoteAddr().(*net.TCPAddr) + msg := strings.Builder{} + if local.IP.IsLoopback() { + msg.WriteString(strconv.Itoa(local.Port)) + } else { + msg.WriteString(local.String()) + } + + msg.WriteRune(' ') + msg.WriteRune(direction) + msg.WriteRune(' ') + + if remote.IP.IsLoopback() { + textPort := c.T.portsRev[uint16(remote.Port)] + if textPort != "" { + msg.WriteString(textPort) + } else { + msg.WriteString(strconv.Itoa(remote.Port)) + } + } else { + msg.WriteString(local.String()) + } + + if _, ok := c.Conn.(*tls.Conn); ok { + msg.WriteString(" [tls]") + } + msg.WriteString(": ") + fmt.Fprintf(&msg, f, args...) + c.T.Log(strings.TrimRight(msg.String(), "\r\n ")) +} + +func (c *Conn) TLS() { + c.T.Helper() + + tlsC := tls.Client(c.Conn, &tls.Config{ + ServerName: "maddy.test", + InsecureSkipVerify: true, + }) + if err := tlsC.Handshake(); err != nil { + c.fatal("TLS handshake fail: %v", err) + } + + c.Conn = tlsC + c.Scanner = bufio.NewScanner(c.Conn) +} + +func (c *Conn) SMTPPlainAuth(username, password string, expectOk bool) { + c.T.Helper() + + resp := append([]byte{0x00}, username...) + resp = append(resp, 0x00) + resp = append(resp, password...) + c.Writeln("AUTH PLAIN " + base64.StdEncoding.EncodeToString(resp)) + if expectOk { + c.ExpectPattern("235 *") + } else { + c.ExpectPattern("*") + } +} + +func (c *Conn) SMTPNegotation(ourName string, requireExts, blacklistExts []string) { + c.T.Helper() + + needCapsMap := make(map[string]bool) + blacklistCapsMap := make(map[string]bool) + for _, ext := range requireExts { + needCapsMap[ext] = false + } + for _, ext := range blacklistExts { + blacklistCapsMap[ext] = false + } + + c.Writeln("EHLO " + ourName) + + // Consume the first line from socket, it is either initial greeting (sent + // before we sent EHLO) or the EHLO reply in case of re-negotiation after + // STARTTLS. + l, err := c.Readln() + if err != nil { + c.T.Fatal("I/O error during SMTP negotiation:", err) + } + if strings.HasPrefix(l, "220") { + // That was initial greeting, consume one more line. + c.ExpectPattern("250-*") + } + + var caps []string +capsloop: + for { + line, err := c.Readln() + if err != nil { + c.T.Fatal("I/O error during SMTP negotiation:", err) + } + + switch { + case strings.HasPrefix(line, "250-"): + caps = append(caps, strings.TrimPrefix(line, "250-")) + case strings.HasPrefix(line, "250 "): + caps = append(caps, strings.TrimPrefix(line, "250 ")) + break capsloop + default: + c.T.Fatal("Unexpected reply during SMTP negotiation:", line) + } + } + + for _, ext := range caps { + needCapsMap[ext] = true + if _, ok := blacklistCapsMap[ext]; ok { + blacklistCapsMap[ext] = true + } + } + for ext, status := range needCapsMap { + if !status { + c.T.Fatalf("Capability %v is missing but required", ext) + } + } + for ext, status := range blacklistCapsMap { + if status { + c.T.Fatalf("Capability %v is present but not allowed", ext) + } + } +} + +func (c *Conn) Close() error { + return c.Conn.Close() +} + +func (c *Conn) Rebind(subtest *T) *Conn { + cpy := *c + cpy.T = subtest + return &cpy +} diff --git a/tests/cover_test.go b/tests/cover_test.go new file mode 100644 index 0000000..298e3e9 --- /dev/null +++ b/tests/cover_test.go @@ -0,0 +1,91 @@ +//go:build cover_main +// +build cover_main + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests + +/* +Go toolchain lacks the ability to instrument arbitrary executables with +coverage counters. + +This file wraps the maddy executable into a minimal layer of "test" logic to +make 'go test' work for it and produce the coverage report. + +Use ./build_cover.sh to compile it into ./maddy.cover. + +References: +https://stackoverflow.com/questions/43381335/how-to-capture-code-coverage-from-a-go-binary +https://blog.cloudflare.com/go-coverage-with-external-tests/ +https://github.com/albertito/chasquid/blob/master/coverage_test.go +*/ + +import ( + "flag" + "io" + "os" + "testing" + + _ "github.com/foxcpp/maddy" // To register run command + _ "github.com/foxcpp/maddy/internal/cli/ctl" // To register other CLI commands. + + maddycli "github.com/foxcpp/maddy/internal/cli" +) + +func TestMain(m *testing.M) { + // -test.* flags are registered somewhere in init() in "testing" (?). + + // maddy.Run changes the working directory, we need to change it back so + // -test.coverprofile writes out profile in the right location. + wd, err := os.Getwd() + if err != nil { + panic(err) + } + + // Skip flag parsing and make flag.Parse no-op so when + // m.Run calls it it will not error out on maddy flags. + args := os.Args + os.Args = []string{"command"} + flag.Parse() + os.Args = args + + code := maddycli.RunWithoutExit() + + if err := os.Chdir(wd); err != nil { + panic(err) + } + + // Silence output produced by "testing" runtime. + r, w, err := os.Pipe() + if err == nil { + os.Stderr = w + os.Stdout = w + } + go func() { + _, _ = io.ReadAll(r) + }() + + // Even though we do not have any tests to run, we need to call out into + // "testing" to make it process flags and produce the coverage report. + m.Run() + + // TestMain doc says we have to exit with a sensible status code on our + // own. + os.Exit(code) +} diff --git a/tests/dovecot_sasl_test.go b/tests/dovecot_sasl_test.go new file mode 100644 index 0000000..af424e8 --- /dev/null +++ b/tests/dovecot_sasl_test.go @@ -0,0 +1,206 @@ +//go:build integration && (darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris) +// +build integration +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// only posix systems ^ + +package tests_test + +import ( + "bufio" + "errors" + "flag" + "io/ioutil" + "os" + "os/exec" + "os/user" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/foxcpp/maddy/tests" +) + +var DovecotExecutable string + +func init() { + flag.StringVar(&DovecotExecutable, "integration.dovecot", "dovecot", "path to dovecot executable for interop tests") +} + +const dovecotConf = `base_dir = $ROOT/run/ +state_dir = $ROOT/lib/ +log_path = /dev/stderr +ssl = no + +default_internal_user = $USER +default_internal_group = $GROUP +default_login_user = $USER + +passdb { + driver = passwd-file + args = $ROOT/passwd +} + +userdb { + driver = passwd-file + args = $ROOT/passwd +} + +service auth { + unix_listener auth { + mode = 0666 + } +} + +# Dovecot refuses to start without protocols, so we need to give it one. +protocols = imap + +service imap-login { + chroot = + inet_listener imap { + address = 127.0.0.1 + port = 0 + } +} + +service anvil { + chroot = +} + +# Turn on debugging information, to help troubleshooting issues. +auth_verbose = yes +auth_debug = yes +auth_debug_passwords = yes +auth_verbose_passwords = yes +mail_debug = yes +` + +const dovecotPasswd = `tester:{plain}123456:1000:1000::/home/user` + +func runDovecot(t *testing.T) (string, *exec.Cmd) { + dovecotExec, err := exec.LookPath(DovecotExecutable) + if err != nil { + if errors.Is(err, exec.ErrNotFound) { + t.Skip("No Dovecot executable found, skipping interop. tests") + } + t.Fatal(err) + } + + tempDir := t.TempDir() + + curUser, err := user.Current() + if err != nil { + t.Fatal(err) + } + curGroup, err := user.LookupGroupId(curUser.Gid) + if err != nil { + t.Fatal(err) + } + + dovecotConf := strings.NewReplacer( + "$ROOT", tempDir, + "$USER", curUser.Username, + "$GROUP", curGroup.Name).Replace(dovecotConf) + err = ioutil.WriteFile(filepath.Join(tempDir, "dovecot.conf"), []byte(dovecotConf), os.ModePerm) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(tempDir, "passwd"), []byte(dovecotPasswd), os.ModePerm) + if err != nil { + t.Fatal(err) + } + + cmd := exec.Command(dovecotExec, "-F", "-c", filepath.Join(tempDir, "dovecot.conf")) + stderr, err := cmd.StderrPipe() + if err != nil { + t.Fatal(err) + } + + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + + ready := make(chan struct{}, 1) + + go func() { + scnr := bufio.NewScanner(stderr) + for scnr.Scan() { + line := scnr.Text() + + // One of messages printed near completing initialization. + if strings.Contains(line, "starting up for imap") { + time.Sleep(500*time.Millisecond) + ready <- struct{}{} + } + + t.Log("dovecot:", line) + } + if err := scnr.Err(); err != nil { + t.Log("stderr I/O error:", err) + } + }() + + <-ready + + return tempDir, cmd +} + +func cleanDovecot(t *testing.T, tempDir string, cmd *exec.Cmd) { + cmd.Process.Signal(syscall.SIGTERM) + if !t.Failed() { + os.RemoveAll(tempDir) + } else { + t.Log("Dovecot directory is not deleted:", tempDir) + } +} + +func TestDovecotSASLClient(tt *testing.T) { + tt.Parallel() + + dovecotDir, cmd := runDovecot(tt) + defer cleanDovecot(tt, dovecotDir, cmd) + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Env("DOVECOT_SASL_SOCK=" + filepath.Join(dovecotDir, "run", "auth-client")) + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + auth dovecot_sasl unix://{env:DOVECOT_SASL_SOCK} + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("localhost", nil, nil) + c.Writeln("AUTH PLAIN AHRlc3QAMTIzNDU2") // 0x00 test 0x00 123456 (invalid user) + c.ExpectPattern("535 *") + c.Writeln("AUTH PLAIN AHRlc3RlcgAxMjM0NQ==") // 0x00 tester 0x00 12345 (invalid password) + c.ExpectPattern("535 *") + c.Writeln("AUTH PLAIN AHRlc3RlcgAxMjM0NTY=") // 0x00 tester 0x00 123456 + c.ExpectPattern("235 *") +} diff --git a/tests/dovecot_sasld_test.go b/tests/dovecot_sasld_test.go new file mode 100644 index 0000000..8fa2fd6 --- /dev/null +++ b/tests/dovecot_sasld_test.go @@ -0,0 +1,202 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "bufio" + "errors" + "flag" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/foxcpp/maddy/tests" +) + +var ChasquidExecutable string + +func init() { + flag.StringVar(&ChasquidExecutable, "integration.chasquid", "chasquid", "path to chasquid executable for interop tests") +} + +const chasquidConf = `smtp_address: "127.0.0.2:44444" +submission_address: "127.0.0.1:44443" + +data_dir: "$ROOT" +mail_log_path: "/dev/null" + +dovecot_auth: true +dovecot_userdb_path: "$AUTH_CLIENT" # needs any Unix socket, not actually used +dovecot_client_path: "$AUTH_CLIENT" +` + +// RSA 1024, valid for *.example.invalid, 127.0.0.1, 127.0.0.2,, 127.0.0.3 +// until Nov 18 17:13:45 2029 GMT. +const testServerCert = `-----BEGIN CERTIFICATE----- +MIICDzCCAXigAwIBAgIRAJ1x+qCW7L+Hs6sRU8BHmWkwDQYJKoZIhvcNAQELBQAw +EjEQMA4GA1UEChMHQWNtZSBDbzAeFw0xOTExMTgxNzEzNDVaFw0yOTExMTUxNzEz +NDVaMBIxEDAOBgNVBAoTB0FjbWUgQ28wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ +AoGBAPINKMyuu3AvzndLDS2/BroA+DRUcAhWPBxMxG1b1BkkHisAZWteKajKmwdO +O13N8HHBRPPOD56AAPLZGNxYLHn6nel7AiH8k40/xC5tDOthqA82+00fwJHDFCnW +oDLOLcO17HulPvfCSWfefc+uee4kajPa+47hutqZH2bGMTXhAgMBAAGjZTBjMA4G +A1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAA +MC4GA1UdEQQnMCWCESouZXhhbXBsZS5pbnZhbGlkhwR/AAABhwR/AAAChwR/AAAD +MA0GCSqGSIb3DQEBCwUAA4GBAGRn3C2NbwR4cyQmTRm5jcaqi1kAYyEu6U8Q9PJW +Q15BXMKUTx2lw//QScK9MH2JpKxDuzWDSvaxZMnTxgri2uiplqpe8ydsWj6Wl0q9 +2XMGJ9LIxTZk5+cyZP2uOolvmSP/q8VFTyk9Udl6KUZPQyoiiDq4rBFUIxUyb+bX +pHkR +-----END CERTIFICATE-----` + +const testServerKey = `-----BEGIN PRIVATE KEY----- +MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAPINKMyuu3AvzndL +DS2/BroA+DRUcAhWPBxMxG1b1BkkHisAZWteKajKmwdOO13N8HHBRPPOD56AAPLZ +GNxYLHn6nel7AiH8k40/xC5tDOthqA82+00fwJHDFCnWoDLOLcO17HulPvfCSWfe +fc+uee4kajPa+47hutqZH2bGMTXhAgMBAAECgYEAgPjSDH3uEdDnSlkLJJzskJ+D +oR58s3R/gvTElSCg2uSLzo3ffF4oBHAwOqxMpabdvz8j5mSdne7Gkp9qx72TtEG2 +wt6uX1tZhm2UTAkInH8IQDthj98P8vAWQsS6HHEIMErsrW2CyUrAt/+o1BRg/hWW +zixA3CLTthhZTJkaUCECQQD5EM16UcTAKfhr3IZppgq+ZsAOMkeCl3XVV9gHo32i +DL6UFAb27BAYyjfcZB1fPou4RszX0Ryu9yU0P5qm6N47AkEA+MpdAPkaPziY0ok4 +e9Tcee6P0mIR+/AHk9GliVX2P74DDoOHyMXOSRBwdb+z2tYjrdjkNEL1Txe+sHny +k/EukwJBAOBqlmqPwNNRPeiaRHZvSSD0XjqsbSirJl48D4gadPoNt66fOQNGAt8D +Xj/z6U9HgQdiq/IOFmVEhT5FzSh1jL8CQQD3Myth8iGQO84tM0c6U3CWfuHMqsEv +0XnV+HNAmHdLMqOa4joi1dh4ZKs5dDdi828UJ/PnsbhI1FEWzLSpJvWdAkAkVWqf +AC/TvWvEZLA6Z5CllyNzZJ7XvtIaNOosxHDolyZ1HMWMlfEb2K2ZXWLy5foKPeoY +Xi3olS9rB0J+Rvjz +-----END PRIVATE KEY-----` + +func runChasquid(t *testing.T, authClientPath string) (string, *exec.Cmd) { + tempDir := t.TempDir() + t.Log("Using", tempDir) + + chasquidConf := strings.NewReplacer( + "$ROOT", tempDir, + "$AUTH_CLIENT", authClientPath).Replace(chasquidConf) + err := ioutil.WriteFile(filepath.Join(tempDir, "chasquid.conf"), []byte(chasquidConf), os.ModePerm) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(tempDir, "certs", "example.org"), os.ModePerm); err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(tempDir, "certs", "example.org", "fullchain.pem"), []byte(testServerCert), os.ModePerm) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(tempDir, "certs", "example.org", "privkey.pem"), []byte(testServerKey), os.ModePerm) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(tempDir, "domains", "example.org"), os.ModePerm); err != nil { + t.Fatal(err) + } + + err = ioutil.WriteFile(filepath.Join(tempDir, "chasquid.conf"), []byte(chasquidConf), os.ModePerm) + if err != nil { + t.Fatal(err) + } + + cmd := exec.Command(ChasquidExecutable, "-v=2", "-config_dir", tempDir) + t.Log("Launching", cmd.String()) + stderr, err := cmd.StderrPipe() + if err != nil { + t.Fatal(err) + } + + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + + ready := make(chan struct{}, 1) + + go func() { + scnr := bufio.NewScanner(stderr) + for scnr.Scan() { + line := scnr.Text() + + // One of messages printed near completing initialization. + if strings.Contains(line, "Loading certificates") { + time.Sleep(1 * time.Second) + ready <- struct{}{} + } + + t.Log("chasquid:", line) + } + if err := scnr.Err(); err != nil { + t.Log("stderr I/O error:", err) + } + }() + + <-ready + + return tempDir, cmd +} + +func cleanChasquid(t *testing.T, tempDir string, cmd *exec.Cmd) { + cmd.Process.Signal(syscall.SIGTERM) + os.RemoveAll(tempDir) +} + +func TestSASLServerWithChasquid(tt *testing.T) { + tt.Parallel() + + _, err := exec.LookPath(ChasquidExecutable) + if err != nil { + if errors.Is(err, exec.ErrNotFound) { + tt.Skip("No chasquid executable found, skipping interop. tests") + } + tt.Fatal(err) + } + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + dovecot_sasld unix://{env:TEST_STATE_DIR}/auth.sock { + auth pass_table static { + # tester@example.org:123456 + entry tester@example.org "bcrypt:$2a$04$0SaXE/WOMBOfk5jyaKjo.OHkioRljdhMznLnYCg1nrksu9iLd51Ri" + } + }`) + t.Run(1) + defer t.Close() + + chasquidDir, cmd := runChasquid(tt, filepath.Join(t.StateDir(), "auth.sock")) + defer cleanChasquid(tt, chasquidDir, cmd) + + c := t.ConnUnnamed(44443) + defer c.Close() + c.SMTPNegotation("localhost", nil, nil) + c.Writeln("STARTTLS") + c.ExpectPattern("220 *") + c.TLS() + c.Writeln("AUTH PLAIN AHRlc3RAZXhhbXBsZS5vcmcAMTIzNDU2") // 0x00 test@example.org 0x00 123456 (invalid user) + c.ExpectPattern("535 *") + c.Writeln("AUTH PLAIN AHRlc3RlckBleGFtcGxlLm9yZwAxMjM0NQ==") // 0x00 tester 0x00 12345 (invalid password) + c.ExpectPattern("535 *") + c.Writeln("AUTH PLAIN AHRlc3RlckBleGFtcGxlLm9yZwAxMjM0NTY=") // 0x00 tester 0x00 123456 + c.ExpectPattern("235 *") +} diff --git a/tests/gocovcat.go b/tests/gocovcat.go new file mode 100644 index 0000000..4018912 --- /dev/null +++ b/tests/gocovcat.go @@ -0,0 +1,92 @@ +//usr/bin/env go run "$0" "$@"; exit $? +// +// From: https://git.lukeshu.com/go/cmd/gocovcat/ +// +//go:build ignore +// +build ignore + +// Copyright 2017 Luke Shumaker +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +// Command gocovcat combines multiple go cover runs, and prints the +// result on stdout. +package main + +import ( + "bufio" + "fmt" + "os" + "sort" + "strconv" + "strings" +) + +func handleErr(err error) { + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +func main() { + modeBool := false + blocks := map[string]int{} + for _, filename := range os.Args[1:] { + file, err := os.Open(filename) + handleErr(err) + buf := bufio.NewScanner(file) + for buf.Scan() { + line := buf.Text() + + if strings.HasPrefix(line, "mode: ") { + m := strings.TrimPrefix(line, "mode: ") + switch m { + case "set": + modeBool = true + case "count", "atomic": + // do nothing + default: + fmt.Fprintf(os.Stderr, "Unrecognized mode: %s\n", m) + os.Exit(1) + } + } else { + sp := strings.LastIndexByte(line, ' ') + block := line[:sp] + cntStr := line[sp+1:] + cnt, err := strconv.Atoi(cntStr) + handleErr(err) + blocks[block] += cnt + } + } + handleErr(buf.Err()) + } + keys := make([]string, 0, len(blocks)) + for key := range blocks { + keys = append(keys, key) + } + sort.Strings(keys) + modeStr := "count" + if modeBool { + modeStr = "set" + } + fmt.Printf("mode: %s\n", modeStr) + for _, block := range keys { + cnt := blocks[block] + if modeBool && cnt > 1 { + cnt = 1 + } + fmt.Printf("%s %d\n", block, cnt) + } +} diff --git a/tests/golangci-noisy.yml b/tests/golangci-noisy.yml new file mode 100644 index 0000000..d6ca6e0 --- /dev/null +++ b/tests/golangci-noisy.yml @@ -0,0 +1,23 @@ +linters: + enable: + - gosimple + - structcheck + - varcheck + - errcheck + - staticcheck + - ineffassign + - deadcode + - typecheck + - govet + - unused + - goimports + - prealloc + - unconvert + - misspell + - whitespace + - nakedret + - dogsled + - godox + - gocyclo + - dupl + - unparam diff --git a/tests/imap_test.go b/tests/imap_test.go new file mode 100644 index 0000000..ba3fabc --- /dev/null +++ b/tests/imap_test.go @@ -0,0 +1,96 @@ +//go:build integration && cgo && !nosqlite3 +// +build integration,cgo,!nosqlite3 + +package tests_test + +import ( + "testing" + + "github.com/foxcpp/maddy/tests" +) + +func TestIMAPEndpointAuthMap(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("imap") + t.Config(` + storage.imapsql test_store { + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth_map email_localpart + auth pass_table static { + entry "user" "bcrypt:$2a$10$E.AuCH3oYbaRrETXfXwc0.4jRAQBbanpZiCfudsJz9bHzLr/qj6ti" # password: 123 + } + storage &test_store + } + `) + t.Run(1) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN user@example.org 123") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) +} + +func TestIMAPEndpointStorageMap(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("imap") + t.Config(` + storage.imapsql test_store { + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + storage_map email_localpart + + auth_map email_localpart + auth pass_table static { + entry "user" "bcrypt:$2a$10$z9SvUwUjkY8wKOWd9IbISeEmbJua2cXRPqw7s2BnLXJuc6pIMPncK" # password: 123 + } + storage &test_store + } + `) + t.Run(1) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN user@example.org 123") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". CREATE testbox") + imapConn.ExpectPattern(". OK *") + + imapConn2 := t.Conn("imap") + defer imapConn2.Close() + imapConn2.ExpectPattern(`\* OK *`) + imapConn2.Writeln(". LOGIN user@example.com 123") + imapConn2.ExpectPattern(". OK *") + imapConn2.Writeln(`. LIST "" "*"`) + imapConn2.Expect(`* LIST (\HasNoChildren) "." INBOX`) + imapConn2.Expect(`* LIST (\HasNoChildren) "." "testbox"`) + imapConn2.ExpectPattern(". OK *") +} diff --git a/tests/imapsql_test.go b/tests/imapsql_test.go new file mode 100644 index 0000000..69f9e7e --- /dev/null +++ b/tests/imapsql_test.go @@ -0,0 +1,259 @@ +//go:build integration && cgo && !nosqlite3 +// +build integration,cgo,!nosqlite3 + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "testing" + "time" + + "github.com/foxcpp/maddy/tests" +) + +// Smoke test to ensure message delivery is handled correctly. + +func TestImapsqlDelivery(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("imap") + t.Port("smtp") + t.Config(` + storage.imapsql test_store { + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth dummy + storage &test_store + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + + deliver_to &test_store + } + `) + t.Run(2) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN testusr@maddy.test 1234") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) + + smtpConn := t.Conn("smtp") + defer smtpConn.Close() + smtpConn.SMTPNegotation("localhost", nil, nil) + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + smtpConn.Writeln("Hi!") + smtpConn.Writeln(".") + smtpConn.ExpectPattern("2*") + + time.Sleep(500 * time.Millisecond) + + imapConn.Writeln(". NOOP") + imapConn.ExpectPattern(`\* 1 EXISTS`) + imapConn.ExpectPattern(`\* 1 RECENT`) + imapConn.ExpectPattern(". OK *") + + imapConn.Writeln(". FETCH 1 (BODY.PEEK[])") + imapConn.ExpectPattern(`\* 1 FETCH (BODY\[\] {*}*`) + imapConn.Expect(`Delivered-To: testusr@maddy.test`) + imapConn.Expect(`Return-Path: `) + imapConn.ExpectPattern(`Received: from localhost (client.maddy.test \[` + tests.DefaultSourceIP.String() + `\]) by maddy.test`) + imapConn.ExpectPattern(` (envelope-sender ) with ESMTP id *; *`) + imapConn.ExpectPattern(` *`) + imapConn.Expect("From: ") + imapConn.Expect("To: ") + imapConn.Expect("Subject: Hi!") + imapConn.Expect("") + imapConn.Expect("Hi!") + imapConn.Expect(")") + imapConn.ExpectPattern(`. OK *`) +} + +func TestImapsqlDeliveryMap(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("imap") + t.Port("smtp") + t.Config(` + storage.imapsql test_store { + delivery_map email_localpart + auth_normalize precis + + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth dummy + storage &test_store + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + + deliver_to &test_store + } + `) + t.Run(2) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN testusr 1234") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) + + smtpConn := t.Conn("smtp") + defer smtpConn.Close() + smtpConn.SMTPNegotation("localhost", nil, nil) + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + smtpConn.Writeln("Hi!") + smtpConn.Writeln(".") + smtpConn.ExpectPattern("2*") + + time.Sleep(500 * time.Millisecond) + + imapConn.Writeln(". NOOP") + imapConn.ExpectPattern(`\* 1 EXISTS`) + imapConn.ExpectPattern(`\* 1 RECENT`) + imapConn.ExpectPattern(". OK *") +} + +func TestImapsqlAuthMap(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("imap") + t.Port("smtp") + t.Config(` + storage.imapsql test_store { + auth_map regexp "(.*)" "$1@maddy.test" + auth_normalize precis + + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth dummy + storage &test_store + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + + deliver_to &test_store + } + `) + t.Run(2) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN testusr 1234") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) + + smtpConn := t.Conn("smtp") + defer smtpConn.Close() + smtpConn.SMTPNegotation("localhost", nil, nil) + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + smtpConn.Writeln("Hi!") + smtpConn.Writeln(".") + smtpConn.ExpectPattern("2*") + + time.Sleep(500 * time.Millisecond) + + imapConn.Writeln(". NOOP") + imapConn.ExpectPattern(`\* 1 EXISTS`) + imapConn.ExpectPattern(`\* 1 RECENT`) + imapConn.ExpectPattern(". OK *") +} diff --git a/tests/issue327_test.go b/tests/issue327_test.go new file mode 100644 index 0000000..b766424 --- /dev/null +++ b/tests/issue327_test.go @@ -0,0 +1,91 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "strconv" + "testing" + "time" + + "github.com/foxcpp/maddy/internal/testutils" + "github.com/foxcpp/maddy/tests" +) + +func TestIssue327(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + tgtPort := t.Port("target") + t.Config(` + hostname mx.maddy.test + tls off + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + deliver_to queue outbound_queue { + target remote { } # it will fail + autogenerated_msg_domain maddy.test + bounce { + check { + spf + } + deliver_to lmtp tcp://127.0.0.1:{env:TEST_PORT_target} + } + } + }`) + t.Run(1) + defer t.Close() + + be, s := testutils.SMTPServer(tt, "127.0.0.1:"+strconv.Itoa(int(tgtPort))) + s.LMTP = true + be.LMTPDataErr = []error{nil, nil} + defer s.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("client.maddy.test", nil, nil) + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") + + for i := 0; i < 5; i++ { + time.Sleep(1 * time.Second) + if len(be.Messages) != 0 { + break + } + } + + if len(be.Messages) != 1 { + t.Fatal("No DSN sent?", len(be.Messages)) + } +} diff --git a/tests/limits_test.go b/tests/limits_test.go new file mode 100644 index 0000000..e219add --- /dev/null +++ b/tests/limits_test.go @@ -0,0 +1,107 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "testing" + + "github.com/foxcpp/maddy/tests" +) + +func TestConcurrencyLimit(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject no + limits { + all concurrency 1 + } + + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + c1 := t.Conn("smtp") + defer c1.Close() + c1.SMTPNegotation("localhost", nil, nil) + c1.Writeln("MAIL FROM:") + c1.ExpectPattern("250 *") + // Down on semaphore. + + c2 := t.Conn("smtp") + defer c2.Close() + c2.SMTPNegotation("localhost", nil, nil) + c1.Writeln("MAIL FROM:") + // Temporary error due to lock timeout. + c1.ExpectPattern("451 *") +} + +func TestPerIPConcurrency(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject no + limits { + ip concurrency 1 + } + + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + c1 := t.Conn("smtp") + defer c1.Close() + c1.SMTPNegotation("localhost", nil, nil) + c1.Writeln("MAIL FROM:") + c1.ExpectPattern("250 *") + // Down on semaphore. + + c3 := t.Conn4("127.0.0.2", "smtp") + defer c3.Close() + c3.SMTPNegotation("localhost", nil, nil) + c3.Writeln("MAIL FROM:") + c3.ExpectPattern("250 *") + // Down on semaphore (different IP). + + c2 := t.Conn("smtp") + defer c2.Close() + c2.SMTPNegotation("localhost", nil, nil) + c1.Writeln("MAIL FROM:") + // Temporary error due to lock timeout. + c1.ExpectPattern("451 *") +} diff --git a/tests/lmtp_test.go b/tests/lmtp_test.go new file mode 100644 index 0000000..8e29a93 --- /dev/null +++ b/tests/lmtp_test.go @@ -0,0 +1,181 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "strconv" + "strings" + "testing" + + "github.com/foxcpp/maddy/internal/testutils" + "github.com/foxcpp/maddy/tests" +) + +func TestLMTPServer_Is_Actually_LMTP(tt *testing.T) { + tt.Parallel() + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("lmtp") + t.Config(` + lmtp tcp://127.0.0.1:{env:TEST_PORT_lmtp} { + hostname mx.maddy.test + tls off + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + c := t.Conn("lmtp") + defer c.Close() + + c.Writeln("LHLO client.maddy.test") + c.ExpectPattern("220 *") +capsloop: + for { + line, err := c.Readln() + if err != nil { + t.Fatal("I/O error:", err) + } + switch { + case strings.HasPrefix(line, "250-"): + case strings.HasPrefix(line, "250 "): + break capsloop + default: + t.Fatal("Unexpected deply:", line) + } + } + + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") +} + +func TestLMTPClient_Is_Actually_LMTP(tt *testing.T) { + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + tgtPort := t.Port("target") + t.Config(` + hostname mx.maddy.test + tls off + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + deliver_to lmtp tcp://127.0.0.1:{env:TEST_PORT_target} + }`) + t.Run(1) + defer t.Close() + + be, s := testutils.SMTPServer(tt, "127.0.0.1:"+strconv.Itoa(int(tgtPort))) + s.LMTP = true + be.LMTPDataErr = []error{nil, nil} + defer s.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("client.maddy.test", nil, nil) + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") + + if be.SessionCounter != 1 { + t.Fatal("No actual connection made?", be.SessionCounter) + } +} + +func TestLMTPClient_Issue308(tt *testing.T) { + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + tgtPort := t.Port("target") + t.Config(` + hostname mx.maddy.test + tls off + + target.lmtp local_mailboxes { + targets tcp://127.0.0.1:{env:TEST_PORT_target} + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + deliver_to &local_mailboxes + }`) + t.Run(1) + defer t.Close() + + be, s := testutils.SMTPServer(tt, "127.0.0.1:"+strconv.Itoa(int(tgtPort))) + s.LMTP = true + be.LMTPDataErr = []error{nil, nil} + defer s.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("client.maddy.test", nil, nil) + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") + + if be.SessionCounter != 1 { + t.Fatal("No actual connection made?", be.SessionCounter) + } +} diff --git a/tests/mta_test.go b/tests/mta_test.go new file mode 100644 index 0000000..21ac001 --- /dev/null +++ b/tests/mta_test.go @@ -0,0 +1,135 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "net" + "strconv" + "testing" + + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/internal/testutils" + "github.com/foxcpp/maddy/tests" +) + +func TestMTA_Outbound(tt *testing.T) { + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + }) + t.Port("smtp") + tgtPort := t.Port("remote_smtp") + t.Config(` + hostname mx.maddy.test + tls off + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + deliver_to remote + }`) + t.Run(1) + defer t.Close() + + be, s := testutils.SMTPServer(tt, "127.0.0.1:"+strconv.Itoa(int(tgtPort))) + defer s.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("client.maddy.test", nil, nil) + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") + + if be.SessionCounter != 1 { + t.Fatal("No actual connection made?", be.SessionCounter) + } +} + +func TestIssue321(tt *testing.T) { + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + "example.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + }) + t.Port("smtp") + tgtPort := t.Port("remote_smtp") + t.Config(` + hostname mx.maddy.test + tls off + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + deliver_to remote { + mx_auth { + dnssec + dane + } + } + }`) + t.Run(1) + defer t.Close() + + be, s := testutils.SMTPServer(tt, "127.0.0.1:"+strconv.Itoa(int(tgtPort))) + defer s.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("client.maddy.test", nil, nil) + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") + + if be.SessionCounter != 1 { + t.Fatal("No actual connection made?", be.SessionCounter) + } +} diff --git a/tests/multiple_domains_test.go b/tests/multiple_domains_test.go new file mode 100644 index 0000000..6b29c3f --- /dev/null +++ b/tests/multiple_domains_test.go @@ -0,0 +1,340 @@ +//go:build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2025 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "testing" + + "github.com/foxcpp/maddy/tests" +) + +// Test cases based on https://maddy.email/multiple-domains/ + +func TestMultipleDomains_SeparateNamespace(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("submission") + t.Port("imap") + t.Config(` + tls off + hostname test.maddy.email + + auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } + } + storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db + } + + submission tcp://0.0.0.0:{env:TEST_PORT_submission} { + auth &local_authdb + reject + } + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + auth &local_authdb + storage &local_mailboxes + } + `) + + t.MustRunCLIGroup( + []string{"creds", "create", "-p", "user1", "user1@test1.maddy.email"}, + []string{"creds", "create", "-p", "user2", "user2@test1.maddy.email"}, + []string{"creds", "create", "-p", "user3", "user1@test2.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user1@test1.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user2@test1.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user1@test2.maddy.email"}, + ) + t.Run(2) + + user1 := t.Conn("imap") + defer user1.Close() + user1.ExpectPattern(`\* OK *`) + user1.Writeln(`. LOGIN user1@test1.maddy.email user1`) + user1.ExpectPattern(`. OK *`) + user1.Writeln(`. CREATE user1`) + user1.ExpectPattern(`. OK *`) + + user1SMTP := t.Conn("submission") + defer user1SMTP.Close() + user1SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user1SMTP.SMTPPlainAuth("user1@test1.maddy.email", "user1", true) + + user2 := t.Conn("imap") + defer user2.Close() + user2.ExpectPattern(`\* OK *`) + user2.Writeln(`. LOGIN user2@test1.maddy.email user2`) + user2.ExpectPattern(`. OK *`) + user2.Writeln(`. CREATE user2`) + user2.ExpectPattern(`. OK *`) + + user2SMTP := t.Conn("submission") + defer user2SMTP.Close() + user2SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user2SMTP.SMTPPlainAuth("user2@test1.maddy.email", "user2", true) + + user3 := t.Conn("imap") + defer user3.Close() + user3.ExpectPattern(`\* OK *`) + user3.Writeln(`. LOGIN user1@test2.maddy.email user3`) + user3.ExpectPattern(`. OK *`) + user3.Writeln(`. CREATE user3`) + user3.ExpectPattern(`. OK *`) + + user3SMTP := t.Conn("submission") + defer user3SMTP.Close() + user3SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user3SMTP.SMTPPlainAuth("user1@test2.maddy.email", "user3", true) + + user1.Writeln(`. LIST "" "*"`) + user1.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user1.Expect(`* LIST (\HasNoChildren) "." "user1"`) + user1.ExpectPattern(". OK *") + + user2.Writeln(`. LIST "" "*"`) + user2.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user2.Expect(`* LIST (\HasNoChildren) "." "user2"`) + user2.ExpectPattern(". OK *") + + user3.Writeln(`. LIST "" "*"`) + user3.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user3.Expect(`* LIST (\HasNoChildren) "." "user3"`) + user3.ExpectPattern(". OK *") +} + +func TestMultipleDomains_SharedCredentials_DistinctMailboxes(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("submission") + t.Port("imap") + t.Config(` + tls off + hostname test.maddy.email + auth_map email_localpart + + auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } + } + storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db + } + + submission tcp://0.0.0.0:{env:TEST_PORT_submission} { + auth &local_authdb + reject + } + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + auth &local_authdb + storage &local_mailboxes + } + `) + + t.MustRunCLIGroup( + []string{"creds", "create", "-p", "user1", "user1"}, + []string{"creds", "create", "-p", "user2", "user2"}, + []string{"imap-acct", "create", "--no-specialuse", "user1@test1.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user2@test1.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user1@test2.maddy.email"}, + ) + t.Run(2) + + user1 := t.Conn("imap") + defer user1.Close() + user1.ExpectPattern(`\* OK *`) + user1.Writeln(`. LOGIN user1@test1.maddy.email user1`) + user1.ExpectPattern(`. OK *`) + user1.Writeln(`. CREATE user1`) + user1.ExpectPattern(`. OK *`) + + user1SMTP := t.Conn("submission") + defer user1SMTP.Close() + user1SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user1SMTP.SMTPPlainAuth("user1@test1.maddy.email", "user1", true) + + user2 := t.Conn("imap") + defer user2.Close() + user2.ExpectPattern(`\* OK *`) + user2.Writeln(`. LOGIN user2@test1.maddy.email user2`) + user2.ExpectPattern(`. OK *`) + user2.Writeln(`. CREATE user2`) + user2.ExpectPattern(`. OK *`) + + user2SMTP := t.Conn("submission") + defer user2SMTP.Close() + user2SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user2SMTP.SMTPPlainAuth("user2@test1.maddy.email", "user2", true) + + user3 := t.Conn("imap") + defer user3.Close() + user3.ExpectPattern(`\* OK *`) + user3.Writeln(`. LOGIN user1@test2.maddy.email user1`) + user3.ExpectPattern(`. OK *`) + user3.Writeln(`. CREATE user3`) + user3.ExpectPattern(`. OK *`) + + user3SMTP := t.Conn("submission") + defer user3SMTP.Close() + user3SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user3SMTP.SMTPPlainAuth("user1@test2.maddy.email", "user1", true) + + user1.Writeln(`. LIST "" "*"`) + user1.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user1.Expect(`* LIST (\HasNoChildren) "." "user1"`) + user1.ExpectPattern(". OK *") + + user2.Writeln(`. LIST "" "*"`) + user2.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user2.Expect(`* LIST (\HasNoChildren) "." "user2"`) + user2.ExpectPattern(". OK *") + + user3.Writeln(`. LIST "" "*"`) + user3.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user3.Expect(`* LIST (\HasNoChildren) "." "user3"`) + user3.ExpectPattern(". OK *") +} + +func TestMultipleDomains_SharedCredentials_SharedMailboxes(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("submission") + t.Port("imap") + t.Config(` + tls off + hostname test.maddy.email + auth_map email_localpart_optional + + auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } + } + storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db + + delivery_map email_localpart_optional + } + + submission tcp://0.0.0.0:{env:TEST_PORT_submission} { + auth &local_authdb + reject + } + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + auth &local_authdb + storage &local_mailboxes + + storage_map email_localpart_optional + } + `) + + t.MustRunCLIGroup( + []string{"creds", "create", "-p", "user1", "user1"}, + []string{"creds", "create", "-p", "user2", "user2"}, + []string{"imap-acct", "create", "--no-specialuse", "user1"}, + []string{"imap-acct", "create", "--no-specialuse", "user2"}, + ) + t.Run(2) + + user1 := t.Conn("imap") + defer user1.Close() + user1.ExpectPattern(`\* OK *`) + user1.Writeln(`. LOGIN user1 user1`) + user1.ExpectPattern(`. OK *`) + user1.Writeln(`. CREATE user1`) + user1.ExpectPattern(`. OK *`) + + user1SMTP := t.Conn("submission") + defer user1SMTP.Close() + user1SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user1SMTP.SMTPPlainAuth("user1", "user1", true) + + user2 := t.Conn("imap") + defer user2.Close() + user2.ExpectPattern(`\* OK *`) + user2.Writeln(`. LOGIN user2@test1.maddy.email user2`) + user2.ExpectPattern(`. OK *`) + user2.Writeln(`. CREATE user2`) + user2.ExpectPattern(`. OK *`) + + user2SMTP := t.Conn("submission") + defer user2SMTP.Close() + user2SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user2SMTP.SMTPPlainAuth("user2", "user2", true) + + user12 := t.Conn("imap") + defer user12.Close() + user12.ExpectPattern(`\* OK *`) + user12.Writeln(`. LOGIN user1@test2.maddy.email user1`) + user12.ExpectPattern(`. OK *`) + user12.Writeln(`. CREATE user12`) + user12.ExpectPattern(`. OK *`) + + user13 := t.Conn("imap") + defer user13.Close() + user13.ExpectPattern(`\* OK *`) + user13.Writeln(`. LOGIN user1@test.maddy.email user1`) + user13.ExpectPattern(`. OK *`) + user13.Writeln(`. CREATE user13`) + user13.ExpectPattern(`. OK *`) + + user12SMTP := t.Conn("submission") + defer user12SMTP.Close() + user12SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user12SMTP.SMTPPlainAuth("user1", "user1", true) + + user13SMTP := t.Conn("submission") + defer user13SMTP.Close() + user13SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user13SMTP.SMTPPlainAuth("user1@test.maddy.email", "user1", true) + + user1.Writeln(`. LIST "" "*"`) + user1.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user1.Expect(`* LIST (\HasNoChildren) "." "user1"`) + user1.Expect(`* LIST (\HasNoChildren) "." "user12"`) + user1.Expect(`* LIST (\HasNoChildren) "." "user13"`) + user1.ExpectPattern(". OK *") + + user2.Writeln(`. LIST "" "*"`) + user2.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user2.Expect(`* LIST (\HasNoChildren) "." "user2"`) + user2.ExpectPattern(". OK *") + + user12.Writeln(`. LIST "" "*"`) + user12.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user12.Expect(`* LIST (\HasNoChildren) "." "user1"`) + user12.Expect(`* LIST (\HasNoChildren) "." "user12"`) + user12.Expect(`* LIST (\HasNoChildren) "." "user13"`) + user12.ExpectPattern(". OK *") +} diff --git a/tests/replace_addr_test.go b/tests/replace_addr_test.go new file mode 100644 index 0000000..c898071 --- /dev/null +++ b/tests/replace_addr_test.go @@ -0,0 +1,245 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "testing" + + "github.com/foxcpp/maddy/tests" +) + +func TestReplaceAddr_Rcpt(tt *testing.T) { + test := func(name, cfg string) { + tt.Run(name, func(tt *testing.T) { + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(cfg) + t.Run(1) + defer t.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("client.maddy.test", nil, nil) + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") + }) + } + + test("inline", ` + hostname mx.maddy.test + tls off + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + modify { + replace_rcpt static { + entry a@maddy.test b@maddy.test + } + } + destination a@maddy.test { + reject + } + destination b@maddy.test { + deliver_to dummy + } + default_destination { + reject + } + }`) + test("inline qualified", ` + hostname mx.maddy.test + tls off + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + modify { + modify.replace_rcpt static { + entry a@maddy.test b@maddy.test + } + } + destination a@maddy.test { + reject + } + destination b@maddy.test { + deliver_to dummy + } + default_destination { + reject + } + }`) + + // FIXME: Not implemented + // test("external", ` + // hostname mx.maddy.test + // tls off + + // modify.replace_rcpt local_aliases { + // table static { + // entry a@maddy.test b@maddy.test + // } + // } + + // smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + // modify { + // &local_aliases + // } + // source a@maddy.test { + // destination a@maddy.test { + // reject + // } + // destination b@maddy.test { + // deliver_to dummy + // } + // default_destination { + // reject + // } + // } + // default_source { + // reject + // } + // }`) +} + +func TestReplaceAddr_Sender(tt *testing.T) { + test := func(name, cfg string) { + tt.Run(name, func(tt *testing.T) { + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(cfg) + t.Run(1) + defer t.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("client.maddy.test", nil, nil) + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") + }) + } + + test("inline", ` + hostname mx.maddy.test + tls off + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + modify { + replace_sender static { + entry a@maddy.test b@maddy.test + } + } + source a@maddy.test { + reject + } + source b@maddy.test { + destination a@maddy.test { + deliver_to dummy + } + default_destination { + reject + } + } + default_source { + reject + } + }`) + test("inline qualified", ` + hostname mx.maddy.test + tls off + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + modify { + modify.replace_sender static { + entry a@maddy.test b@maddy.test + } + } + source a@maddy.test { + reject + } + source b@maddy.test { + destination a@maddy.test { + deliver_to dummy + } + default_destination { + reject + } + } + default_source { + reject + } + }`) + // FIXME: Not implemented + // test("external", ` + // hostname mx.maddy.test + // tls off + + // modify.replace_sender local_aliases { + // table static { + // entry a@maddy.test b@maddy.test + // } + // } + + // smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + // modify { + // &local_aliases + // } + // source a@maddy.test { + // reject + // } + // source b@maddy.test { + // destination a@maddy.test { + // deliver_to dummy + // } + // default_destination { + // reject + // } + // } + // default_source { + // reject + // } + // }`) +} diff --git a/tests/run.sh b/tests/run.sh new file mode 100755 index 0000000..afbc802 --- /dev/null +++ b/tests/run.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +if [ -z "$GO" ]; then + export GO=go +fi + +./build_cover.sh + +clean() { + rm -f /tmp/maddy-coverage-report* +} +trap clean EXIT + +$GO test -tags integration -integration.executable ./maddy.cover -integration.coverprofile /tmp/maddy-coverage-report "$@" +$GO run gocovcat.go /tmp/maddy-coverage-report* > coverage.out diff --git a/tests/smtp_autobuffer_test.go b/tests/smtp_autobuffer_test.go new file mode 100644 index 0000000..58ef341 --- /dev/null +++ b/tests/smtp_autobuffer_test.go @@ -0,0 +1,347 @@ +//go:build integration && cgo && !nosqlite3 +// +build integration,cgo,!nosqlite3 + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "strings" + "testing" + "time" + + "github.com/foxcpp/maddy/tests" +) + +func TestSMTPEndpoint_LargeMessage(tt *testing.T) { + // Send 1.44 MiB message to verify it being handled correctly + // everywhere. + // (Issue 389) + + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("imap") + t.Port("smtp") + t.Config(` + storage.imapsql test_store { + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth dummy + storage &test_store + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + + deliver_to &test_store + } + `) + t.Run(2) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN testusr@maddy.test 1234") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) + + smtpConn := t.Conn("smtp") + defer smtpConn.Close() + smtpConn.SMTPNegotation("localhost", nil, nil) + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + for i := 0; i < 3000; i++ { + smtpConn.Writeln(strings.Repeat("A", 500)) + } + // 3000*502 ~ 1.44 MiB not including header side + smtpConn.Writeln(".") + + time.Sleep(500 * time.Millisecond) + + imapConn.Writeln(". NOOP") + imapConn.ExpectPattern(`\* 1 EXISTS`) + imapConn.ExpectPattern(`\* 1 RECENT`) + imapConn.ExpectPattern(". OK *") + + imapConn.Writeln(". FETCH 1 (BODY.PEEK[])") + imapConn.ExpectPattern(`\* 1 FETCH (BODY\[\] {1506312}*`) + imapConn.Expect(`Delivered-To: testusr@maddy.test`) + imapConn.Expect(`Return-Path: `) + imapConn.ExpectPattern(`Received: from localhost (client.maddy.test \[` + tests.DefaultSourceIP.String() + `\]) by maddy.test`) + imapConn.ExpectPattern(` (envelope-sender ) with ESMTP id *; *`) + imapConn.ExpectPattern(` *`) + imapConn.Expect("From: ") + imapConn.Expect("To: ") + imapConn.Expect("Subject: Hi!") + imapConn.Expect("") + for i := 0; i < 3000; i++ { + imapConn.Expect(strings.Repeat("A", 500)) + } + imapConn.Expect(")") + imapConn.ExpectPattern(`. OK *`) +} + +func TestSMTPEndpoint_FileBuffer(tt *testing.T) { + run := func(tt *testing.T, bufferOpt string) { + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("imap") + t.Port("smtp") + t.Config(` + storage.imapsql test_store { + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth dummy + storage &test_store + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + buffer ` + bufferOpt + ` + + deliver_to &test_store + } + `) + t.Run(2) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN testusr@maddy.test 1234") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) + + smtpConn := t.Conn("smtp") + defer smtpConn.Close() + smtpConn.SMTPNegotation("localhost", nil, nil) + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + smtpConn.Writeln("AAAAABBBBBB") + smtpConn.Writeln(".") + smtpConn.ExpectPattern("2*") + + time.Sleep(500 * time.Millisecond) + + imapConn.Writeln(". NOOP") + imapConn.ExpectPattern(`\* 1 EXISTS`) + imapConn.ExpectPattern(`\* 1 RECENT`) + imapConn.ExpectPattern(". OK *") + + imapConn.Writeln(". FETCH 1 (BODY.PEEK[])") + imapConn.ExpectPattern(`\* 1 FETCH (BODY\[\] {*}*`) + imapConn.Expect(`Delivered-To: testusr@maddy.test`) + imapConn.Expect(`Return-Path: `) + imapConn.ExpectPattern(`Received: from localhost (client.maddy.test \[` + tests.DefaultSourceIP.String() + `\]) by maddy.test`) + imapConn.ExpectPattern(` (envelope-sender ) with ESMTP id *; *`) + imapConn.ExpectPattern(` *`) + imapConn.Expect("From: ") + imapConn.Expect("To: ") + imapConn.Expect("Subject: Hi!") + imapConn.Expect("") + imapConn.Expect("AAAAABBBBBB") + imapConn.Expect(")") + imapConn.ExpectPattern(`. OK *`) + } + + tt.Run("ram", func(tt *testing.T) { run(tt, "ram") }) + tt.Run("fs", func(tt *testing.T) { run(tt, "fs") }) +} + +func TestSMTPEndpoint_Autobuffer(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("imap") + t.Port("smtp") + t.Config(` + storage.imapsql test_store { + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth dummy + storage &test_store + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + buffer auto 5b + + deliver_to &test_store + } + `) + t.Run(2) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN testusr@maddy.test 1234") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) + + smtpConn := t.Conn("smtp") + defer smtpConn.Close() + smtpConn.SMTPNegotation("localhost", nil, nil) + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + smtpConn.Writeln("AAAAABBBBBB") + smtpConn.Writeln(".") + smtpConn.ExpectPattern("2*") + + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + smtpConn.Writeln(".") + smtpConn.ExpectPattern("2*") + + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + smtpConn.Writeln("AAA") + smtpConn.Writeln(".") + smtpConn.ExpectPattern("2*") + + time.Sleep(500 * time.Millisecond) + + imapConn.Writeln(". NOOP") + // This will break with go-imap v2 upgrade merging updates. + imapConn.ExpectPattern(`\* 3 EXISTS`) + imapConn.ExpectPattern(`\* 3 RECENT`) + imapConn.ExpectPattern(". OK *") + + imapConn.Writeln(". FETCH 1:3 (BODY.PEEK[])") + imapConn.ExpectPattern(`\* 1 FETCH (BODY\[\] {*}*`) + imapConn.Expect(`Delivered-To: testusr@maddy.test`) + imapConn.Expect(`Return-Path: `) + imapConn.ExpectPattern(`Received: from localhost (client.maddy.test \[` + tests.DefaultSourceIP.String() + `\]) by maddy.test`) + imapConn.ExpectPattern(` (envelope-sender ) with ESMTP id *; *`) + imapConn.ExpectPattern(` *`) + imapConn.Expect("From: ") + imapConn.Expect("To: ") + imapConn.Expect("Subject: Hi!") + imapConn.Expect("") + imapConn.Expect("AAAAABBBBBB") + imapConn.Expect(")") + imapConn.ExpectPattern(`\* 2 FETCH (BODY\[\] {*}*`) + imapConn.Expect(`Delivered-To: testusr@maddy.test`) + imapConn.Expect(`Return-Path: `) + imapConn.ExpectPattern(`Received: from localhost (client.maddy.test \[` + tests.DefaultSourceIP.String() + `\]) by maddy.test`) + imapConn.ExpectPattern(` (envelope-sender ) with ESMTP id *; *`) + imapConn.ExpectPattern(` *`) + imapConn.Expect("From: ") + imapConn.Expect("To: ") + imapConn.Expect("Subject: Hi!") + imapConn.Expect("") + imapConn.Expect(")") + imapConn.ExpectPattern(`\* 3 FETCH (BODY\[\] {*}*`) + imapConn.Expect(`Delivered-To: testusr@maddy.test`) + imapConn.Expect(`Return-Path: `) + imapConn.ExpectPattern(`Received: from localhost (client.maddy.test \[` + tests.DefaultSourceIP.String() + `\]) by maddy.test`) + imapConn.ExpectPattern(` (envelope-sender ) with ESMTP id *; *`) + imapConn.ExpectPattern(` *`) + imapConn.Expect("From: ") + imapConn.Expect("To: ") + imapConn.Expect("Subject: Hi!") + imapConn.Expect("") + imapConn.Expect("AAA") + imapConn.Expect(")") + imapConn.ExpectPattern(`. OK *`) +} diff --git a/tests/smtp_test.go b/tests/smtp_test.go new file mode 100644 index 0000000..d897227 --- /dev/null +++ b/tests/smtp_test.go @@ -0,0 +1,667 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "errors" + "fmt" + "io/ioutil" + "path/filepath" + "strings" + "testing" + + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/tests" +) + +func TestCheckRequireTLS(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls self_signed + + defer_sender_reject no + + check { + require_tls + } + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.SMTPNegotation("localhost", nil, nil) + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("550 5.7.1 *") + conn.Writeln("STARTTLS") + conn.ExpectPattern("220 *") + conn.TLS() + conn.SMTPNegotation("localhost", nil, nil) + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("250 *") + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestProxyProtocolTrustedSource(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + "one.maddy.test.": { + TXT: []string{"v=spf1 ip4:127.0.0.17 -all"}, + }, + }) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + proxy_protocol { + trust ` + tests.DefaultSourceIP.String() + ` ::1/128 + tls off + } + + defer_sender_reject no + + check { + spf { + enforce_early yes + fail_action reject + } + } + + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.Writeln(fmt.Sprintf("PROXY TCP4 127.0.0.17 %s 12345 %d", tests.DefaultSourceIP.String(), t.Port("smtp"))) + conn.SMTPNegotation("localhost", nil, nil) + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("250 *") + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestProxyProtocolUntrustedSource(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + "one.maddy.test.": { + TXT: []string{"v=spf1 ip4:127.0.0.17 -all"}, + }, + }) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + proxy_protocol { + trust fe80::bad/128 + tls off + } + + defer_sender_reject no + + check { + spf { + enforce_early yes + fail_action reject + } + } + + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.Writeln(fmt.Sprintf("PROXY TCP4 127.0.0.17 %s 12345 %d", tests.DefaultSourceIP.String(), t.Port("smtp"))) + conn.SMTPNegotation("localhost", nil, nil) + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("550 *") + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestCheckSPF(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + "none.maddy.test.": { + TXT: []string{}, + }, + "pass.maddy.test.": { + TXT: []string{"v=spf1 +all"}, + }, + "neutral.maddy.test.": { + TXT: []string{"v=spf1 ?all"}, + }, + "fail.maddy.test.": { + TXT: []string{"v=spf1 -all"}, + }, + "softfail.maddy.test.": { + TXT: []string{"v=spf1 ~all"}, + }, + "permerr.maddy.test.": { + TXT: []string{"v=spf1 something_clever"}, + }, + "temperr.maddy.test.": { + Err: errors.New("IANA forgot to resign the root zone"), + }, + }) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject no + + check { + spf { + enforce_early yes + + none_action reject 551 + neutral_action reject + fail_action reject 552 + softfail_action reject 553 + permerr_action reject 554 + temperr_action reject 455 + } + } + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.SMTPNegotation("fail.maddy.test", nil, nil) + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("250 *") + conn.Writeln("RSET") + conn.ExpectPattern("250 *") + + // Actually checks fail.maddy.test. + conn.Writeln("MAIL FROM:<>") + conn.ExpectPattern("552 5.7.0 *") + + conn.SMTPNegotation("pass.maddy.test", nil, nil) + + conn.Writeln("MAIL FROM:<>") + conn.ExpectPattern("250 *") + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("551 5.7.0 *") + + // Also check the default enhanced code is meaningful. + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("550 5.7.23 *") + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("552 5.7.0 *") + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("553 5.7.0 *") + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("554 5.7.0 *") + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("455 4.7.0 *") + + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestSPF_DMARCDefer(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + "subdomain.maddy-dmarc.test.": { + TXT: []string{"v=spf1 -all"}, + }, + "maddy-dmarc.test.": { + TXT: []string{"v=spf1 -all"}, + }, + "_dmarc.maddy-dmarc.test.": { + TXT: []string{"v=DMARC1; p=reject; sp=none"}, + }, + "subdomain.maddy-dmarc2.test.": { + TXT: []string{"v=spf1 -all"}, + }, + "maddy-dmarc2.test.": { + TXT: []string{"v=spf1 -all"}, + }, + "_dmarc.maddy-dmarc2.test.": { + TXT: []string{"v=DMARC1; p=reject"}, + }, + "maddy-no-dmarc.test.": { + TXT: []string{"v=spf1 -all"}, + }, + "maddy-dmarc-lookup-fail.test.": { + TXT: []string{"v=spf1 -all"}, + }, + "_dmarc.maddy-dmarc-lookup-fail.test.": { + Err: errors.New("nop"), + }, + }) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject no + + check { + spf { + enforce_early no + + none_action ignore + neutral_action reject + fail_action reject + softfail_action reject + permerr_action reject + temperr_action reject + } + } + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.SMTPNegotation("localhost", nil, nil) + + msg := func(fromEnv, fromHdr, bodyRespPattern string) { + tt.Helper() + + conn.Writeln("MAIL FROM:<" + fromEnv + ">") + conn.ExpectPattern("250 *") + conn.Writeln("RCPT TO:") + conn.ExpectPattern("250 *") + conn.Writeln("DATA") + conn.ExpectPattern("354 *") + conn.Writeln("From: <" + fromHdr + ">") + conn.Writeln("") + conn.Writeln("Heya!") + conn.Writeln(".") + conn.ExpectPattern(bodyRespPattern) + } + + msg("test@subdomain.maddy-dmarc.test", "test@subdomain.maddy-dmarc.test", "550 *") + + // Malformed From domain, DMARC cannot work so use only SPF. + msg("test@subdomain.maddy-dmarc.test", "", "550 *") + + msg("test@subdomain.maddy-dmarc.test", "maddy-dmarc-lookup-fail.test", "550 *") + + // No actual DMARC check is done but SPF check results are not applied. + msg("test@maddy-dmarc.test", "test@maddy-dmarc.test", "250 *") + msg("test@maddy-dmarc2.test", "test@maddy-dmarc2.test", "250 *") + + msg("test@maddy-no-dmarc.test", "test@maddy-no-dmarc.test", "550 *") + + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestDNSBLConfig(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + tests.DefaultSourceIPRev + ".dnsbl.test.": { + A: []string{"127.0.0.127"}, + }, + "sender.test.dnsbl.test.": { + A: []string{"127.0.0.127"}, + }, + }) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject no + + check { + dnsbl { + reject_threshold 1 + + dnsbl.test { + client_ipv4 + mailfrom + } + } + } + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.SMTPNegotation("localhost", nil, nil) + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("554 5.7.0 Client identity is listed in the used DNSBL *") + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("554 5.7.0 Client identity is listed in the used DNSBL *") + + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestDNSBLConfig2(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + tests.DefaultSourceIPRev + ".dnsbl2.test.": { + A: []string{"127.0.0.127"}, + }, + "sender.test.dnsbl.test.": { + A: []string{"127.0.0.127"}, + }, + }) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject no + + check { + dnsbl { + reject_threshold 1 + + dnsbl.test { + mailfrom + } + dnsbl2.test { + client_ipv4 + score -1 + } + } + } + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.SMTPNegotation("localhost", nil, nil) + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("250 *") + + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestCheckAuthorizeSender(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + auth dummy + defer_sender_reject off + + source example1.org { + check { + authorize_sender { + auth_normalize precis_casefold + user_to_email static { + entry "test-user1" "test@example1.org" + entry "test-user2" "é@example1.org" + } + } + } + deliver_to dummy + } + source example2.org { + check { + authorize_sender { + auth_normalize precis_casefold + prepare_email static { + entry "alias-to-test@example2.org" "test@example2.org" + } + user_to_email static { + entry "test-user1" "test@example2.org" + entry "test-user2" "test2@example2.org" + } + } + } + deliver_to dummy + } + + default_source { + reject + } + }`) + t.Run(1) + defer t.Close() + + c := t.Conn("smtp") + c.SMTPNegotation("client.maddy.test", nil, nil) + c.SMTPPlainAuth("test-user2", "1", true) + c.Writeln("MAIL FROM:") + c.ExpectPattern("5*") // rejected - user is not test-user1 + c.Writeln("MAIL FROM:") + c.ExpectPattern("5*") // rejected - unknown email + c.Writeln("MAIL FROM: SMTPUTF8") + c.ExpectPattern("2*") // OK - é@example1.org belongs to test-user2 + c.Close() + + c = t.Conn("smtp") + c.SMTPNegotation("client.maddy.test", nil, nil) + c.SMTPPlainAuth("test-user1", "1", true) + c.Writeln("MAIL FROM:") + c.ExpectPattern("5*") // rejected - user is not test-user2 + c.Writeln("MAIL FROM:") + c.ExpectPattern("2*") // OK - test@example2.org belongs to test-user + c.Writeln("MAIL FROM:") + c.ExpectPattern("2*") // OK - test@example2.org belongs to test-user + c.Close() +} + +func TestCheckCommand(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + check { + command {env:TEST_PWD}/testdata/check_command.sh {sender} { + code 12 reject + } + } + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.SMTPNegotation("localhost", nil, nil) + + // Note: Internally, messages are handled using LF line endings, being + // converted CRLF only when transfered over Internet protocols. + expectedMsg := "From: \n" + + "To: \n" + + "Subject: Hi there!\n" + + "\n" + + "Nice to meet you!\n" + submitMsg := func(conn *tests.Conn, from string) { + // Fairly trivial SMTP transaction. + conn.Writeln("MAIL FROM:<" + from + ">") + conn.ExpectPattern("250 *") + conn.Writeln("RCPT TO:") + conn.ExpectPattern("250 *") + conn.Writeln("DATA") + conn.ExpectPattern("354 *") + conn.Writeln("From: ") + conn.Writeln("To: ") + conn.Writeln("Subject: Hi there!") + conn.Writeln("") + conn.Writeln("Nice to meet you!") + conn.Writeln(".") + } + + t.Subtest("Message dump", func(t *tests.T) { + conn := conn.Rebind(t) + + submitMsg(conn, "testing@maddy.test") + conn.ExpectPattern("250 *") + + msgPath := filepath.Join(t.StateDir(), "msg") + msgContents, err := ioutil.ReadFile(msgPath) + if err != nil { + t.Fatal(err) + } + + if string(msgContents) != expectedMsg { + t.Log("Wrong message contents received by check script!") + t.Log("Actual:") + t.Log(msgContents) + t.Log("Expected:") + t.Log(expectedMsg) + } + }) + t.Subtest("Message dump + Add header", func(t *tests.T) { + conn := conn.Rebind(t) + + submitMsg(conn, "testing+addHeader@maddy.test") + conn.ExpectPattern("250 *") + + msgPath := filepath.Join(t.StateDir(), "msg") + msgContents, err := ioutil.ReadFile(msgPath) + if err != nil { + t.Fatal(err) + } + + expectedMsg := "X-Added-Header: 1\n" + expectedMsg + if string(msgContents) != expectedMsg { + t.Log("Wrong message contents received by check script!") + t.Log("Actual:") + t.Log(msgContents) + t.Log("Expected:") + t.Log(expectedMsg) + } + }) + t.Subtest("Body reject", func(t *tests.T) { + conn := conn.Rebind(t) + + submitMsg(conn, "testing+reject@maddy.test") + conn.ExpectPattern("550 *") + + msgPath := filepath.Join(t.StateDir(), "msg") + msgContents, err := ioutil.ReadFile(msgPath) + if err != nil { + t.Fatal(err) + } + + if string(msgContents) != expectedMsg { + t.Log("Wrong message contents received by check script!") + t.Log("Actual:") + t.Log(msgContents) + t.Log("Expected:") + t.Log([]byte(expectedMsg)) + } + }) + + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestHeaderSizeConstraint(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + deliver_to dummy + max_header_size 1K + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.SMTPNegotation("localhost", nil, nil) + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("250 *") + conn.Writeln("RCPT TO:") + conn.ExpectPattern("250 *") + conn.Writeln("DATA") + conn.ExpectPattern("354 *") + conn.Writeln("From: ") + conn.Writeln("To: ") + conn.Writeln("Subject: " + strings.Repeat("A", 2*1024)) + conn.Writeln("") + conn.Writeln("Hi") + conn.Writeln(".") + + conn.ExpectPattern("552 5.3.4 Message header size exceeds limit *") + + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} diff --git a/tests/stress_test.go b/tests/stress_test.go new file mode 100644 index 0000000..74ed268 --- /dev/null +++ b/tests/stress_test.go @@ -0,0 +1,411 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "sync" + "testing" + "time" + + "github.com/foxcpp/maddy/tests" +) + +func floodSmtp(c *tests.Conn, commands, expectedPatterns []string, iterations int) { + for i := 0; i < iterations; i++ { + for i, cmd := range commands { + c.Writeln(cmd) + if expectedPatterns[i] != "" { + c.ExpectPattern(expectedPatterns[i]) + } + } + } +} + +func TestSMTPFlood_FullMsg_NoLimits_1Conn(tt *testing.T) { + tt.Parallel() + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "DATA", + "From: ", + "", + "Heya!", + ".", + }, []string{ + "250 *", + "250 *", + "354 *", + "", + "", + "", + "250 *", + }, 100) +} + +func TestSMTPFlood_FullMsg_NoLimits_10Conns(tt *testing.T) { + tt.Parallel() + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + wg := sync.WaitGroup{} + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "DATA", + "From: ", + "", + "Heya!", + ".", + }, []string{ + "250 *", + "250 *", + "354 *", + "", + "", + "", + "250 *", + }, 100) + t.Log("Done") + }() + } + + wg.Wait() +} + +func TestSMTPFlood_EnvelopeAbort_NoLimits_10Conns(tt *testing.T) { + tt.Parallel() + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + wg := sync.WaitGroup{} + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "RSET", + }, []string{ + "250 *", + "250 *", + "250 *", + }, 100) + t.Log("Done") + }() + } + + wg.Wait() +} + +func TestSMTPFlood_EnvelopeAbort_Ratelimited(tt *testing.T) { + tt.Parallel() + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + limits { + all rate 10 1s + } + + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + conns := 5 + msgsPerConn := 10 + expectedRate := 10 + slip := 10 + + start := time.Now() + + wg := sync.WaitGroup{} + for i := 0; i < conns; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "RSET", + }, []string{ + "250 *", + "250 *", + "250 *", + }, msgsPerConn) + t.Log("Done") + }() + } + + wg.Wait() + end := time.Now() + + t.Log("Sent", conns*msgsPerConn, "messages using", conns, "connections") + t.Log("Took", end.Sub(start)) + + effectiveRate := float64(conns*msgsPerConn) / end.Sub(start).Seconds() + if effectiveRate > float64(expectedRate+slip) { + t.Fatal("Effective rate is significantly bigger than limit:", effectiveRate) + } + t.Log("Effective rate:", effectiveRate) +} + +func TestSMTPFlood_FullMsg_Ratelimited_PerSource(tt *testing.T) { + tt.Parallel() + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject false + + limits { + source rate 10 1s + } + + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + conns := 5 + msgsPerConn := 10 + expectedRate := 10 + slip := 10 + + start := time.Now() + + wg := sync.WaitGroup{} + for i := 0; i < conns; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "DATA", + "From: ", + "", + "Heya!", + ".", + }, []string{ + "250 *", + "250 *", + "354 *", + "", + "", + "", + "250 *", + }, msgsPerConn) + t.Log("Done") + }() + } + for i := 0; i < conns; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "DATA", + "From: ", + "", + "Heya!", + ".", + }, []string{ + "250 *", + "250 *", + "354 *", + "", + "", + "", + "250 *", + }, msgsPerConn) + t.Log("Done") + }() + } + + wg.Wait() + end := time.Now() + + t.Log("Sent", conns*msgsPerConn, "messages using", conns, "connections") + t.Log("Took", end.Sub(start)) + + effectiveRate := float64(conns*msgsPerConn*2) / end.Sub(start).Seconds() + // Expect the rate twice since we send from two sources. + if effectiveRate > float64(expectedRate*2+slip) { + t.Fatal("Effective rate is significantly bigger than limit:", effectiveRate) + } + t.Log("Effective rate:", effectiveRate) +} + +func TestSMTPFlood_EnvelopeAbort_Ratelimited_PerIP(tt *testing.T) { + tt.Parallel() + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject false + + limits { + ip rate 10 1s + } + + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + conns := 2 + msgsPerConn := 50 + expectedRate := 10 + slip := 10 + + start := time.Now() + + wg := sync.WaitGroup{} + for i := 0; i < conns; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := t.Conn4("127.0.0.1", "smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "RSET", + }, []string{ + "250 *", + "250 *", + "250 *", + }, msgsPerConn) + t.Log("Done") + }() + } + for i := 0; i < conns; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := t.Conn4("127.0.0.2", "smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "RSET", + }, []string{ + "250 *", + "250 *", + "250 *", + }, msgsPerConn) + t.Log("Done") + }() + } + + wg.Wait() + end := time.Now() + + t.Log("Sent", 2*conns*msgsPerConn, "messages using", conns*2, "connections") + t.Log("Took", end.Sub(start)) + + effectiveRate := float64(conns*msgsPerConn*2) / end.Sub(start).Seconds() + // Expect the rate twice since we send from two sources. + if effectiveRate > float64(expectedRate*2+slip) { + t.Fatal("Effective rate is significantly bigger than limit:", effectiveRate) + } + t.Log("Expected rate:", expectedRate*2) + t.Log("Effective rate:", effectiveRate) +} diff --git a/tests/t.go b/tests/t.go new file mode 100644 index 0000000..c1c04bc --- /dev/null +++ b/tests/t.go @@ -0,0 +1,484 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package tests provides the framework for integration testing of maddy. +// +// The packages core object is tests.T object that encapsulates all test +// state. It runs the server using test-provided configuration file and acts as +// a proxy for all interactions with the server. +package tests + +import ( + "bufio" + "bytes" + "flag" + "fmt" + "math/rand" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/foxcpp/go-mockdns" +) + +var ( + TestBinary = "./maddy" + CoverageOut string + DebugLog bool +) + +type T struct { + *testing.T + + testDir string + cfg string + + dnsServ *mockdns.Server + env []string + ports map[string]uint16 + portsRev map[uint16]string + + servProc *exec.Cmd +} + +func NewT(t *testing.T) *T { + return &T{ + T: t, + ports: map[string]uint16{}, + portsRev: map[uint16]string{}, + } +} + +// Config sets the configuration to use for the server. It must be called +// before Run. +func (t *T) Config(cfg string) { + t.Helper() + + if t.servProc != nil { + panic("tests: Config called after Run") + } + + t.cfg = cfg +} + +// DNS sets the DNS zones to emulate for the tested server instance. +// +// If it is not called before Run, DNS(nil) call is assumed which makes the +// mockdns server respond with NXDOMAIN to all queries. +func (t *T) DNS(zones map[string]mockdns.Zone) { + t.Helper() + + if zones == nil { + zones = map[string]mockdns.Zone{} + } + if _, ok := zones["100.97.109.127.in-addr.arpa."]; !ok { + zones["100.97.109.127.in-addr.arpa."] = mockdns.Zone{PTR: []string{"client.maddy.test."}} + } + + if t.dnsServ != nil { + t.Log("NOTE: Multiple DNS calls, replacing the server instance...") + t.dnsServ.Close() + } + + dnsServ, err := mockdns.NewServer(zones, false) + if err != nil { + t.Fatal("Test configuration failed:", err) + } + dnsServ.Log = t + t.dnsServ = dnsServ +} + +// Port allocates the random TCP port for use by test. It will made accessible +// in the configuration via environment variables with name in the form +// TEST_PORT_name. +// +// If there is a port with name remote_smtp, it will be passed as the value for +// the -debug.smtpport parameter. +func (t *T) Port(name string) uint16 { + if port := t.ports[name]; port != 0 { + return port + } + + // TODO: Try to bind on port to test its usability. + port := rand.Int31n(45536) + 20000 + t.ports[name] = uint16(port) + t.portsRev[uint16(port)] = name + return uint16(port) +} + +func (t *T) Env(kv string) { + t.env = append(t.env, kv) +} + +func (t *T) ensureCanRun() { + if t.cfg == "" { + panic("tests: Run called without configuration set") + } + if t.dnsServ == nil { + // If there is no DNS zones set in test - start a server that will + // respond with NXDOMAIN to all queries to avoid accidentally leaking + // any DNS queries to the real world. + t.Log("NOTE: Explicit DNS(nil) is recommended.") + t.DNS(nil) + + t.Cleanup(func() { + // Shutdown the DNS server after maddy to make sure it will not spend time + // timing out queries. + if err := t.dnsServ.Close(); err != nil { + t.Log("Unable to stop the DNS server:", err) + } + t.dnsServ = nil + }) + } + + // Setup file system, create statedir, runtimedir, write out config. + if t.testDir == "" { + testDir, err := os.MkdirTemp("", "maddy-tests-") + if err != nil { + t.Fatal("Test configuration failed:", err) + } + t.testDir = testDir + t.Log("using", t.testDir) + + if err := os.MkdirAll(filepath.Join(t.testDir, "statedir"), os.ModePerm); err != nil { + t.Fatal("Test configuration failed:", err) + } + if err := os.MkdirAll(filepath.Join(t.testDir, "runtimedir"), os.ModePerm); err != nil { + t.Fatal("Test configuration failed:", err) + } + + t.Cleanup(func() { + if !t.Failed() { + return + } + + t.Log("removing", t.testDir) + os.RemoveAll(t.testDir) + t.testDir = "" + }) + } + + configPreable := "state_dir " + filepath.Join(t.testDir, "statedir") + "\n" + + "runtime_dir " + filepath.Join(t.testDir, "runtimedir") + "\n\n" + + err := os.WriteFile(filepath.Join(t.testDir, "maddy.conf"), []byte(configPreable+t.cfg), os.ModePerm) + if err != nil { + t.Fatal("Test configuration failed:", err) + } +} + +func (t *T) buildCmd(additionalArgs ...string) *exec.Cmd { + // Assigning 0 by default will make outbound SMTP unusable. + remoteSmtp := "0" + if port := t.ports["remote_smtp"]; port != 0 { + remoteSmtp = strconv.Itoa(int(port)) + } + + args := []string{"-config", filepath.Join(t.testDir, "maddy.conf"), + "-debug.smtpport", remoteSmtp, + "-debug.dnsoverride", t.dnsServ.LocalAddr().String(), + "-log", "/tmp/test.log"} + + if CoverageOut != "" { + args = append(args, "-test.coverprofile", CoverageOut+"."+strconv.FormatInt(time.Now().UnixNano(), 16)) + } + if DebugLog { + args = append(args, "-debug") + } + + args = append(args, additionalArgs...) + + cmd := exec.Command(TestBinary, args...) + + pwd, err := os.Getwd() + if err != nil { + t.Fatal("Test configuration failed:", err) + } + + // Set environment variables. + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, + "TEST_PWD="+pwd, + "TEST_STATE_DIR="+filepath.Join(t.testDir, "statedir"), + "TEST_RUNTIME_DIR="+filepath.Join(t.testDir, "runtimedir"), + ) + for name, port := range t.ports { + cmd.Env = append(cmd.Env, fmt.Sprintf("TEST_PORT_%s=%d", name, port)) + } + cmd.Env = append(cmd.Env, t.env...) + + return cmd +} + +func (t *T) MustRunCLIGroup(args ...[]string) { + t.ensureCanRun() + + wg := sync.WaitGroup{} + for _, arg := range args { + wg.Add(1) + go func() { + defer wg.Done() + + _, err := t.RunCLI(arg...) + if err != nil { + t.Printf("maddy %v: %v", arg, err) + t.Fail() + } + }() + } + wg.Wait() +} + +func (t *T) MustRunCLI(args ...string) string { + s, err := t.RunCLI(args...) + if err != nil { + t.Fatalf("maddy %v: %v", args, err) + } + return s +} + +func (t *T) RunCLI(args ...string) (string, error) { + t.ensureCanRun() + cmd := t.buildCmd(args...) + + var stderr, stdout bytes.Buffer + cmd.Stderr = &stderr + cmd.Stdout = &stdout + + t.Log("launching maddy", cmd.Args) + if err := cmd.Run(); err != nil { + t.Log("Stderr:", stderr.String()) + t.Fatal("Test configuration failed:", err) + } + + t.Log("Stderr:", stderr.String()) + + return stdout.String(), nil +} + +// Run completes the configuration of test environment and starts the test server. +// +// T.Close should be called by the end of test to release any resources and +// shutdown the server. +// +// The parameter waitListeners specifies the amount of listeners the server is +// supposed to configure. Run() will block before all of them are up. +func (t *T) Run(waitListeners int) { + t.ensureCanRun() + cmd := t.buildCmd("run") + + // Capture maddy log and redirect it. + logOut, err := cmd.StderrPipe() + if err != nil { + t.Fatal("Test configuration failed:", err) + } + + t.Log("launching maddy", cmd.Args) + if err := cmd.Start(); err != nil { + t.Fatal("Test configuration failed:", err) + } + + // Log scanning goroutine checks for the "listening" messages and sends 'true' + // on the channel each time. + listeningMsg := make(chan bool) + + go func() { + defer logOut.Close() + defer close(listeningMsg) + scnr := bufio.NewScanner(logOut) + for scnr.Scan() { + line := scnr.Text() + + if strings.Contains(line, "listening on") { + listeningMsg <- true + line += " (test runner>listener wait trigger<)" + } + + t.Log("maddy:", line) + } + if err := scnr.Err(); err != nil { + t.Log("stderr I/O error:", err) + } + }() + + for i := 0; i < waitListeners; i++ { + if !<-listeningMsg { + t.Fatal("Log ended before all expected listeners are up. Start-up error?") + } + } + + t.servProc = cmd + + t.Cleanup(t.killServer) +} + +func (t *T) StateDir() string { + return filepath.Join(t.testDir, "statedir") +} + +func (t *T) RuntimeDir() string { + return filepath.Join(t.testDir, "statedir") +} + +func (t *T) killServer() { + if err := t.servProc.Process.Signal(os.Interrupt); err != nil { + t.Log("Unable to kill the server process:", err) + os.RemoveAll(t.testDir) + return // Return, as now it is pointless to wait for it. + } + + go func() { + time.Sleep(5 * time.Second) + if t.servProc != nil { + t.Log("Killing possibly hung server process") + t.servProc.Process.Kill() //nolint:errcheck + } + }() + + if err := t.servProc.Wait(); err != nil { + t.Error("The server did not stop cleanly, deadlock?") + } + + t.servProc = nil + + if err := os.RemoveAll(t.testDir); err != nil { + t.Log("Failed to remove test directory:", err) + } + t.testDir = "" +} + +func (t *T) Close() { + t.Log("close is no-op") +} + +// Printf implements Logger interfaces used by some libraries. +func (t *T) Printf(f string, a ...interface{}) { + t.Logf(f, a...) +} + +// Conn6 connects to the server listener at the specified named port using IPv6 loopback. +func (t *T) Conn6(portName string) Conn { + port := t.ports[portName] + if port == 0 { + panic("tests: connection for the unused port name is requested") + } + + conn, err := net.Dial("tcp6", "[::1]:"+strconv.Itoa(int(port))) + if err != nil { + t.Fatal("Could not connect, is server listening?", err) + } + + return Conn{ + T: t, + WriteTimeout: 1 * time.Second, + ReadTimeout: 15 * time.Second, + Conn: conn, + Scanner: bufio.NewScanner(conn), + } +} + +// Conn4 connects to the server listener at the specified named port using one +// of 127.0.0.0/8 addresses as a source. +func (t *T) Conn4(sourceIP, portName string) Conn { + port := t.ports[portName] + if port == 0 { + panic("tests: connection for the unused port name is requested") + } + + localIP := net.ParseIP(sourceIP) + if localIP == nil { + panic("tests: invalid localIP argument") + } + if localIP.To4() == nil { + panic("tests: only IPv4 addresses are allowed") + } + + conn, err := net.DialTCP("tcp4", &net.TCPAddr{ + IP: localIP, + Port: 0, + }, &net.TCPAddr{ + IP: net.IPv4(127, 0, 0, 1), + Port: int(port), + }) + if err != nil { + t.Fatal("Could not connect, is server listening?", err) + } + + return Conn{ + T: t, + WriteTimeout: 1 * time.Second, + ReadTimeout: 15 * time.Second, + Conn: conn, + Scanner: bufio.NewScanner(conn), + } +} + +var ( + DefaultSourceIP = net.IPv4(127, 109, 97, 100) + DefaultSourceIPRev = "100.97.109.127" +) + +func (t *T) ConnUnnamed(port uint16) Conn { + conn, err := net.DialTCP("tcp4", &net.TCPAddr{ + IP: DefaultSourceIP, + Port: 0, + }, &net.TCPAddr{ + IP: net.IPv4(127, 0, 0, 1), + Port: int(port), + }) + if err != nil { + t.Fatal("Could not connect, is server listening?", err) + } + + return Conn{ + T: t, + WriteTimeout: 1 * time.Second, + ReadTimeout: 15 * time.Second, + Conn: conn, + Scanner: bufio.NewScanner(conn), + } +} + +func (t *T) Conn(portName string) Conn { + port := t.ports[portName] + if port == 0 { + panic("tests: connection for the unused port name is requested") + } + + return t.ConnUnnamed(port) +} + +func (t *T) Subtest(name string, f func(t *T)) { + t.T.Run(name, func(subTT *testing.T) { + subT := *t + subT.T = subTT + f(&subT) + }) +} + +func init() { + flag.StringVar(&TestBinary, "integration.executable", "./maddy", "executable to test") + flag.StringVar(&CoverageOut, "integration.coverprofile", "", "write coverage stats to file (requires special maddy executable)") + flag.BoolVar(&DebugLog, "integration.debug", false, "pass -debug to maddy executable") +} diff --git a/tests/testdata/check_command.sh b/tests/testdata/check_command.sh new file mode 100755 index 0000000..4284b3c --- /dev/null +++ b/tests/testdata/check_command.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +if [ -e "${TEST_PWD}/testdata/${1}.hdr" ]; then + cat "${TEST_PWD}/testdata/${1}.hdr" +fi + +cat > ${TEST_STATE_DIR}/msg + +if [ -e "${TEST_PWD}/testdata/${1}.exit" ]; then + exit "$(cat "${TEST_PWD}/testdata/${1}.exit")" +fi diff --git a/tests/testdata/testing+addHeader@maddy.test.hdr b/tests/testdata/testing+addHeader@maddy.test.hdr new file mode 100644 index 0000000..71b07a4 --- /dev/null +++ b/tests/testdata/testing+addHeader@maddy.test.hdr @@ -0,0 +1 @@ +X-Added-Header: 1 diff --git a/tests/testdata/testing+reject@maddy.test.exit b/tests/testdata/testing+reject@maddy.test.exit new file mode 100644 index 0000000..48082f7 --- /dev/null +++ b/tests/testdata/testing+reject@maddy.test.exit @@ -0,0 +1 @@ +12