commit 1910e01da1f237d92c641d8a760deb45a0f1b860 Author: yann Date: Fri Sep 5 16:54:41 2025 +0200 first commit diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ef99e5a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +testdata/ +cmd/maddy/maddy +maddy +tests/maddy.cover diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..3ee4163 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{scd,go}] +indent_style = tab +indent_size = 4 + +[*.yml] +indent_style = tab +indent_size = 2 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..3108941 --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,52 @@ +# Code of Merit + +**1.** The project creators, lead developers, core team, constitute the managing +members of the project and have final say in every decision of the project, +technical or otherwise, including overruling previous decisions. There are no +limitations to this decisional power. + +**2.** Contributions are an expected result of your membership on the project. +Don’t expect others to do your work or help you with your work forever. + +**3.** All members have the same opportunities to seek any challenge they want +within the project. + +**4.** Authority or position in the project will be proportional to the accrued +contribution. Seniority must be earned. + +**5.** Software is evolutive: the better implementations must supersede lesser +implementations. Technical advantage is the primary evaluation metric. + +**6.** This is a space for technical prowess; topics outside of the project will +not be tolerated. + +**7.** Non technical conflicts will be discussed in a separate space. Disruption +of the project will not be allowed. + +**8.** Individual characteristics, including but not limited to, body, sex, +sexual preference, race, language, religion, nationality, or political +preferences are irrelevant in the scope of the project and will not be taken +into account concerning your value or that of your contribution to the project. + +**9.** Discuss or debate the idea, not the person. + +**10.** There is no room for ambiguity: Ambiguity will be met with questioning; +further ambiguity will be met with silence. It is the responsibility of the +originator to provide requested context. + +**11.** If something is illegal outside the scope of the project, it is illegal +in the scope of the project. This Code of Merit does not take precedence over +governing law. + +**12.** This Code of Merit governs the technical procedures of the project not +the activities outside of it. + +**13.** Participation on the project equates to agreement of this Code of Merit. + +**14.** No objectives beyond the stated objectives of this project are relevant +to the project. Any intent to deviate the project from its original purpose of +existence will constitute grounds for remedial action which may include +expulsion from the project. + +This document is the Code of Merit +(`http://code-of-merit.org`), version 1.0. diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..45bafd9 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,59 @@ +# Contributing Guidelines + +Of course, we love our contributors. Thanks for spending time on making maddy +better. + +## Reporting bugs + +**Issue tracker is meant to be used only if you have a problem or a feature +request. If you just have some questions about maddy - prefer to use the +[IRC channel](https://webchat.oftc.net/?channels=maddy&uio=MT11bmRlZmluZWQb1).** + +- Provide log files, preferably with 'debug' directive set. +- Provide the exact steps to reproduce the issue. +- Provide the example message that causes the error, if applicable. +- "Too much information is better than not enough information". + +Issues without enough information will be ignored and possibly closed. +Take some time to be more useful. + +See SECURITY.md for information on how to report vulnerabilities. + +## Contributing Code + +0. Use common sense. +1. Learn Git. Especially, what is `git rebase`. We may ask you to use it if + needed. +2. Tell us that you are willing to work on an issue. +3. Fork the repo. Create a new branch based on `dev`, write your code. Open a + PR. + +Ask for advice if you are not sure. We don't bite. + +maddy design summary and some recommendations are provided in +[HACKING.md](../HACKING.md) file. + +## Commits + +1. Prefix commit message with a package path if it affects only a single + package. Omit `internal/` for brevity. +2. Provide reasoning for details in the source code itself (via comments), + provide reasoning for high-level decisions in the commit message. +3. Make sure every commit builds & passes tests. Otherwise `git bisect` becomes + unusable. + +## Git workflow + +`dev` branch includes the in-development version for the next feature release. +It is based on commit of the latest stable release and is merged into `master` +on release via fast-forward. Unlike `master`, `dev` **is not a protected branch +and may get force-pushes**. + +`master` branch contains the latest stable release and is frozen between +releases. + +`fix-X.Y` are temporary branches containing backported security fixes. +They are based on the commit of the corresponding stable release and exist +while the corresponding release is maintained. A `fix-*` branch is not created +for the latest release. Changes are added to these branches by cherry-picking +needed commits from the `dev` branch. diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 0000000..310954d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,27 @@ +--- +name: Bug report +about: If you think something is broken +title: Bug report +labels: bug +assignees: '' + +--- + +# Describe the bug + +What do you think is wrong? + +# Steps to reproduce + +# Log files + +Use a service like hastebin.com or attach a file if it is big + +# Configuration file + +Located in /etc/maddy/maddy.conf by default, don't forget to remove DB passwords +and other security-related stuff. + +# Environment information + +* maddy version: ? diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..7573e18 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,7 @@ +contact_links: + - name: Questions + url: "https://github.com/foxcpp/maddy/discussions/new?category=q-a" + about: "Use GitHub discussions for any questions" + - name: IRC channel + url: "https://webchat.oftc.net/?channels=maddy&uio=MT11bmRlZmluZWQb1" + about: "... or there is also an IRC channel for any discussions" diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 0000000..63e29a9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,22 @@ +--- +name: Feature request +about: If you would like to see a new feature in maddy. +title: Feature request +labels: new feature +assignees: '' + +--- + +# Use case + +What problem you are trying to solve? + +Note alternatives you considered and why they are not useful. + +# Your idea for a solution + +How your solution would work in general? +Note that some overly complicated solutions may be rejected because maddy is +meant to be simple. + +- [ ] I'm willing to help with the implementation diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 0000000..150da57 --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,17 @@ +# Security Policy + +## Supported Versions + +Two latest incompatible releases (e.g. 2.0.0 and 1.9.0). + +Latest release gets all bug fixes, features, etc. Previous incompatible release +gets security fixes and fixes for problems that render software completely +unusable in certain configurations with no workaround. + +## Reporting a Vulnerability + +If you believe the vulnerabilitiy does have a big impact on existing +deployments - email `fox.cpp at disroot.org`, put "[maddy security]" in the +Subject. + +Otherwise, open a public issue. diff --git a/.github/releases.md b/.github/releases.md new file mode 100644 index 0000000..b52fa81 --- /dev/null +++ b/.github/releases.md @@ -0,0 +1,41 @@ +# Release preparation + +1. Run linters, fix all simple warnings. If the behavior is intentional - add +`nolint` comment and explanation. If the warning is non-trviail to fix - open +an issue. +``` +golangci-lint run +``` + +2. Run unit tests suite. Verify that all disabled tests are not related to + serious problems and have corresponding issue open. +``` +go test ./... +``` + +3. Run integration tests suite. Verify that all disabled tests are not related + to serious problems and have corresponding issue open. +``` +cd tests/ +./run.sh +``` + +4. Write release notes. + +5. Create PGP-signed Git tag and push it to GitHub (do not create a "release" + yet). + +5. Use environment configuration from maddy-repro bundle + (https://foxcpp.dev/maddy-repro) to build release artifacts. + +6. Create detached PGP signatures for artifacts using key + 3197BBD95137E682A59717B434BB2007081396F4. + +7. Create sha256sums file for artifacts. + +8. Create release on GitHub using the same text for + release notes. Attach signed artifacts and sha256sums file. + +9. Build the Docker container and push it to hub.docker.com. + +10. Post a message on the sr.ht mailing list. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..e7155b6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,158 @@ +name: "Prepare release artifacts" + +on: + push: + tags: [ "v*" ] + +permissions: + id-token: write + contents: read + attestations: write + packages: write + +jobs: + artifact-builder-x86: + name: "Prepare release artifacts (x86)" + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + container: + image: "alpine:edge" + steps: + - uses: actions/checkout@v1 # v2 does not work with containers + - name: "Install build dependencies" + run: | + apk add --no-cache gcc go zstd + - name: "Create and package build tree" + run: | + ./build.sh --builddir ~/package-output/ --static build + ver=$(cat .version) + if [ "v$ver" != "${{github.ref_name}}" ]; then echo ".version does not match the Git tag"; exit 1; fi + mv ~/package-output/ ~/maddy-$ver-x86_64-linux-musl + cd ~ + tar c ./maddy-$ver-x86_64-linux-musl | zstd > ~/maddy-x86_64-linux-musl.tar.zst + cd - + - name: "Save source tree" + run: | + rm -rf .git + ver=$(cat .version) + cp -r . ~/maddy-$ver-src + cd ~ + tar c ./maddy-$ver-src | zstd > ~/maddy-src.tar.zst + cd - + - name: "Upload source tree" + uses: actions/upload-artifact@v4 + with: + name: maddy-src.tar.zst + path: '~/maddy-src.tar.zst' + if-no-files-found: error + - name: "Upload binary tree" + uses: actions/upload-artifact@v4 + with: + name: maddy-binary.tar.zst + path: '~/maddy-x86_64-linux-musl.tar.zst' + if-no-files-found: error + - name: "Generate artifact attestation" + uses: actions/attest-build-provenance@v2 + with: + subject-path: '~/maddy-x86_64-linux-musl.tar.zst' + artifact-builder-arm: + name: "Prepare release artifacts (aarch64)" + if: github.ref_type == 'tag' + runs-on: ubuntu-22.04-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + # Building in a Docker container is a workaround for the issue of + # JavaScript-based GitHub Actions not being supported in Alpine + # containers on the Arm64 platform. Otherwise, we could completely reuse + # artifact-builder-x86 as a matrix job by running it on an Arm runner. + - name: Build in Docker container + run: | + # Create Dockerfile for the build + cat > Dockerfile << 'EOF' + FROM alpine:edge + RUN apk add --no-cache gcc go zstd musl-dev scdoc + WORKDIR /build + COPY . . + RUN ./build.sh --builddir /package-output/ --static build && \ + ver=$(cat .version) && \ + if [ "v$ver" != "${{github.ref_name}}" ]; then echo ".version does not match the Git tag"; exit 1; fi && \ + mv /package-output/ /maddy-$ver-aarch64-linux-musl && \ + cd / && \ + tar c ./maddy-$ver-aarch64-linux-musl | zstd > /maddy-aarch64-linux-musl.tar.zst + EOF + # Build the image, create a temporary container and copy the artifact. + docker build -t maddy-builder . + container_id=$(docker create maddy-builder) + docker cp $container_id:/maddy-aarch64-linux-musl.tar.zst . + docker rm $container_id + - name: Upload binary tree + uses: actions/upload-artifact@v4 + with: + name: maddy-binary-aarch64.tar.zst + path: maddy-aarch64-linux-musl.tar.zst + if-no-files-found: error + - name: "Generate artifact attestation" + uses: actions/attest-build-provenance@v2 + with: + subject-path: 'maddy-aarch64-linux-musl.tar.zst' + docker-builder: + name: "Build & push Docker image" + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: "Set up QEMU" + uses: docker/setup-qemu-action@v1 + with: + platforms: arm64 + - name: "Set up Docker Buildx" + id: buildx + uses: docker/setup-buildx-action@v3 + - name: "Login to Docker Hub" + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + logout: false + - name: "Login to GitHub Container Registry" + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + logout: false # https://news.ycombinator.com/item?id=28607735 + - name: "Generate container metadata" + uses: docker/metadata-action@v5 + id: meta + with: + images: | + foxcpp/maddy + ghcr.io/foxcpp/maddy + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + labels: | + org.opencontainers.image.title=Maddy Mail Server + org.opencontainers.image.documentation=https://maddy.email/docker/ + org.opencontainers.image.url=https://maddy.email + - name: "Build and push" + uses: docker/build-push-action@v6 + id: docker + with: + context: . + platforms: linux/amd64 #,linux/arm64 Temporary disabled due to SIGSEGV in gcc. + file: Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + - name: "Generate container attestation" + uses: actions/attest-build-provenance@v2 + with: + subject-name: ghcr.io/foxcpp/maddy + subject-digest: ${{ steps.docker.outputs.digest }} + push-to-registry: true + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..32c02e5 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,75 @@ +name: "Testing" + +on: + push: + branches: [ master, dev ] + tags: [ "v*" ] + pull_request: + branches: [ master, dev ] + +permissions: + contents: read + pull-requests: read + checks: write + +jobs: + golangci: + name: Lint + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: "Install libpam" + run: | + sudo apt-get update + sudo apt-get install -y libpam-dev + - uses: golangci/golangci-lint-action@v6 + with: + version: v1.60 + args: "--timeout=30m" + buildsh: + name: "Verify build.sh" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: "Install libpam" + run: | + sudo apt-get update + sudo apt-get install -y libpam-dev + - name: "Verify build.sh" + run: | + ./build.sh + ./build.sh --destdir destdir/ install + find destdir/ + test: + name: "Build and test" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: "Install libpam" + run: | + sudo apt-get update + sudo apt-get install -y libpam-dev + - name: "Unit & module tests" + run: | + go test ./... -coverprofile=coverage.out -covermode=atomic + - name: "Integration tests" + run: | + cd tests/ + ./run.sh + - uses: codecov/codecov-action@v2 + with: + files: ./coverage.out + flags: unit + - uses: codecov/codecov-action@v2 + with: + files: ./tests/coverage.out + flags: integration diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..790b848 --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# gitignore.io +*.o +*.a +*.so +_obj +_test +*.[568vq] +[568vq].out +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* +_testmain.go +*.exe +*.exe~ +*.test +*.prof +**/.envrc +**/.DS_Store + +# Tests coverage +*.out + +# Compiled binaries +cmd/maddy/maddy +cmd/maddy-*-helper/maddy-*-helper +/maddy + +# Man pages +docs/man/*.1 +docs/man/*.5 + +# Certificates and private keys. +*.pem +*.crt +*.key + +# Some directories that may be created during test-runs +# in repo directory. +cmd/maddy/*mtasts-cache +cmd/maddy/*queue + +build/ + +tests/maddy.cover +tests/maddy + +.idea/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..8617544 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,17 @@ +linters: + enable: + - gosimple + - errcheck + - staticcheck + - ineffassign + - typecheck + - govet + - unused + - goimports + - prealloc + - unconvert + - misspell + - whitespace + - nakedret + - dogsled + - copyloopvar diff --git a/.mkdocs.yml b/.mkdocs.yml new file mode 100644 index 0000000..3e90eec --- /dev/null +++ b/.mkdocs.yml @@ -0,0 +1,84 @@ +site_name: maddy + +repo_url: https://github.com/foxcpp/maddy + +theme: alb + +markdown_extensions: + - codehilite: + guess_lang: false + +nav: + - faq.md + - Tutorials: + - tutorials/setting-up.md + - tutorials/building-from-source.md + - tutorials/alias-to-remote.md + - tutorials/pam.md + - Release builds: 'https://maddy.email/builds/' + - multiple-domains.md + - upgrading.md + - seclevels.md + - docker.md + - Reference manual: + - reference/modules.md + - reference/global-config.md + - reference/tls.md + - reference/tls-acme.md + - Endpoints configuration: + - reference/endpoints/imap.md + - reference/endpoints/smtp.md + - reference/endpoints/openmetrics.md + - IMAP storage: + - reference/storage/imap-filters.md + - reference/storage/imapsql.md + - Blob storage: + - reference/blob/fs.md + - reference/blob/s3.md + - reference/smtp-pipeline.md + - SMTP targets: + - reference/targets/queue.md + - reference/targets/remote.md + - reference/targets/smtp.md + - SMTP checks: + - reference/checks/actions.md + - reference/checks/dkim.md + - reference/checks/spf.md + - reference/checks/milter.md + - reference/checks/rspamd.md + - reference/checks/dnsbl.md + - reference/checks/command.md + - reference/checks/authorize_sender.md + - reference/checks/misc.md + - SMTP modifiers: + - reference/modifiers/dkim.md + - reference/modifiers/envelope.md + - Lookup tables (string translation): + - reference/table/static.md + - reference/table/regexp.md + - reference/table/file.md + - reference/table/sql_query.md + - reference/table/chain.md + - reference/table/email_localpart.md + - reference/table/email_with_domain.md + - reference/table/auth.md + - Authentication providers: + - reference/auth/pass_table.md + - reference/auth/pam.md + - reference/auth/shadow.md + - reference/auth/external.md + - reference/auth/ldap.md + - reference/auth/dovecot_sasl.md + - reference/auth/plain_separate.md + - reference/auth/netauth.md + - reference/config-syntax.md + - Integration with software: + - third-party/dovecot.md + - third-party/smtp-servers.md + - third-party/rspamd.md + - third-party/mailman3.md + - Internals: + - internals/specifications.md + - internals/unicode.md + - internals/quirks.md + - internals/sqlite.md diff --git a/.version b/.version new file mode 100644 index 0000000..6f4eebd --- /dev/null +++ b/.version @@ -0,0 +1 @@ +0.8.1 diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2da6211 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM golang:1.23-alpine AS build-env + +ARG ADDITIONAL_BUILD_TAGS="" + +RUN set -ex && \ + apk upgrade --no-cache --available && \ + apk add --no-cache build-base + +WORKDIR /maddy + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . ./ +RUN mkdir -p /pkg/data && \ + cp maddy.conf.docker /pkg/data/maddy.conf && \ + ./build.sh --builddir /tmp --destdir /pkg/ --tags "docker ${ADDITIONAL_BUILD_TAGS}" build install + +FROM alpine:3.21.2 +LABEL maintainer="fox.cpp@disroot.org" +LABEL org.opencontainers.image.source=https://github.com/foxcpp/maddy + +RUN set -ex && \ + apk upgrade --no-cache --available && \ + apk --no-cache add ca-certificates +COPY --from=build-env /pkg/data/maddy.conf /data/maddy.conf +COPY --from=build-env /pkg/usr/local/bin/maddy /bin/ + +EXPOSE 25 143 993 587 465 +VOLUME ["/data"] +ENTRYPOINT ["/bin/maddy", "-config", "/data/maddy.conf"] +CMD ["run"] diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 0000000..1394edb --- /dev/null +++ b/HACKING.md @@ -0,0 +1,130 @@ +## Design goals + +- **Make it easy to deploy.** + Minimal configuration changes should be required to get a typical mail server + running. Though, it is important to avoid making guesses for a + "zero-configuration". A wrong guess is worse than no guess. + +- **Provide 80% of needed components.** + E-mail has evolved into a huge mess. With a single package to do one thing, it + quickly turns into a maintenance nightmare. Put all stuff mail server + typically needs into a single package. Though, leave controversial or highly + opinionated stuff out, don't force people to do things our way + (see next point). + +- **Interoperate with existing software.** + Implement (de-facto) standard protocols not only for clients but also for + various server-side helper software (content filters, etc). + +- **Be secure but interoperable.** + Verify DKIM signatures by default, use DMRAC policies by default, etc. This + makes default setup as secure as possible while maintaining reasonable + interoperability. Though, users can configure maddy to be stricter. + +- **Achieve flexibility through composability.** + Allow connecting components in arbitrary ways instead of restricting users to + predefined templates. + +- **Use Go concurrency features to the full extent.** + Do as much I/O as possible in parallel to minimize latencies. It is silly to + not do so when it is possible. + +## Design summary + +Here is a summary of how things are organized in maddy in general. It explains +things from the developer perspective and is meant to be used as an +introduction by the new developers/contributors. It is recommended to read +user documentation to understand how things work from the user perspective as +well. + +- User documentation: [maddy.conf(5)](docs/man/maddy.5.scd) +- Design rationale: [Comments on design (Wiki)][1] + +There are components called "modules". They are represented by objects +implementing the module.Module interface. Each module gets its unique name. +The function used to create a module instance is saved with this name as a key +into the global map called "modules registry". + +Whenever module needs another module for some functionality, it references it +using a configuration directive with a matcher that internally calls +`modconfig.ModuleFromNode`. That function looks up the module "constructor" in +the registry, calls it with corresponding arguments, checks whether the +returned module satisfies the needed interfaces and then initializes it. + +Alternatively, if configuration uses &-syntax to reference existing +configuration block, `ModuleFromNode` simply looks it up in the global instances +registry. All modules defined the configuration as a separate top-level blocks +are created before main initialization and are placed in the instances registry +where they can be looked up as mentioned before. + +Top-level defined module instances are initialized (`Init` method) lazily as +they are required by other modules. 'smtp' and 'imap' modules follow a special +initialization path, so they are always initialized directly. + +## Error handling + +Familiarize yourself with the `github.com/foxcpp/maddy/framework/exterrors` +package and make sure you have the following for returned errors: +- SMTP status information (smtp\_code, smtp\_enchcode, smtp\_msg fields) + - SMTP message text should contain a generic description of the error + condition without any details to prevent accidental disclosure of the + server configuration details. +- `Temporary() == true` for temporary errors (see `exterrors.WithTemporary`) +- Field that includes the module name + +The easiest way to get all of these is to use `exterrors.SMTPError`. +Put the original error into the `Err` field, so it can be inspected using +`errors.Is`, `errors.Unwrap`, etc. Put the module name into `CheckName` or +`TargetName`. Add any additional context information using the `Misc` field. +Note, the SMTP status code overrides the result of `exterrors.IsTemporary()` +for that error object, so set it using `exterrors.SMTPCode` that uses +`IsTemporary` to select between two codes. + +If the error you are wrapping contains details in its structure fields (like +`*net.OpError`) - copy these values into `Misc` map, put the underlying error +object (`net.OpError.Err`, for example) into the `Err` field. +Avoid using `Reason` unless you are sure you can provide the error message +better than the `Err.Error()` or `Err` is `nil`. + +Do not attempt to add a SMTP status information for every single possible +error. Use `exterrors.WithFields` with basic information for errors you don't +expect. The SMTP client will get the "Internal server error" message and this +is generally the right thing to do on unexpected errors. + +### Goroutines and panics + +If you start any goroutines - make sure to catch panics to make sure severe +bugs will not bring the whole server down. + +## Adding a check + +"Check" is a module that inspects the message and flags it as spam or rejects +it altogether based on some condition. + +The skeleton for the stateful check module can be found in +`internal/check/skeleton.go`. Throw it into a file in +`internal/check/check_name` directory and start ~~breaking~~ extending it. + +If you don't need any per-message state, you can use `StatelessCheck` wrapper. +See `check/dns` directory for a working example. + +Here are some guidelines to make sure your check works well: +- RTFM, docs will tell you about any caveats. +- Don't share any state _between_ messages, your code will be executed in + parallel. +- Use `github.com/foxcpp/maddy/check.FailAction` to select behavior on check + failures. See other checks for examples on how to use it. +- You can assume that order of check functions execution is as follows: + `CheckConnection`, `CheckSender`, `CheckRcpt`, `CheckBody`. + +## Adding a modifier + +"Modifier" is a module that can modify some parts of the message data. + +Note, currently this is not possible to modify the body contents, only header +can be modified. + +Structure of the modifier implementation is similar to the structure of check +implementation, check `modify/replace\_addr.go` for a working example. + +[1]: https://github.com/foxcpp/maddy/wiki/Dev:-Comments-on-design diff --git a/README.md b/README.md new file mode 100644 index 0000000..312fb84 --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +Maddy Mail Server +===================== +> Composable all-in-one mail server. + +Maddy Mail Server implements all functionality required to run a e-mail +server. It can send messages via SMTP (works as MTA), accept messages via SMTP +(works as MX) and store messages while providing access to them via IMAP. +In addition to that it implements auxiliary protocols that are mandatory +to keep email reasonably secure (DKIM, SPF, DMARC, DANE, MTA-STS). + +It replaces Postfix, Dovecot, OpenDKIM, OpenSPF, OpenDMARC and more with one +daemon with uniform configuration and minimal maintenance cost. + +**Note:** IMAP storage is "beta". If you are looking for stable and +feature-packed implementation you may want to use Dovecot instead. maddy still +can handle message delivery business. + +[![CI status](https://img.shields.io/github/actions/workflow/status/foxcpp/maddy/cicd.yml?style=flat-square)](https://github.com/foxcpp/maddy/actions/workflows/cicd.yml) +[![Issues tracker](https://img.shields.io/github/issues/foxcpp/maddy?style=flat-square)](https://github.com/foxcpp/maddy) + +* [Setup tutorial](https://maddy.email/tutorials/setting-up/) +* [Documentation](https://maddy.email/) + +* [IRC channel](https://webchat.oftc.net/?channels=maddy&uio=MT11bmRlZmluZWQb1) +* [Mailing list](https://lists.sr.ht/~foxcpp/maddy) diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..419dc1f --- /dev/null +++ b/build.sh @@ -0,0 +1,201 @@ +#!/bin/sh + +destdir=/ +builddir="$PWD/build" +prefix=/usr/local +version= +static=0 +if [ "${GOFLAGS}" = "" ]; then + GOFLAGS="-trimpath" # set some flags to avoid passing "" to go +fi + +print_help() { + cat >&2 < build tags to use + --version version tag to embed into executables (default: auto-detect) + +Additional flags for "go build" can be provided using GOFLAGS environment variable. + +Options for ./build.sh install: + --prefix installation prefix (default: $prefix) + --destdir system root (default: $destdir) +EOF +} + +while :; do + case "$1" in + -h|--help) + print_help + exit + ;; + --builddir) + shift + builddir="$1" + ;; + --prefix) + shift + prefix="$1" + ;; + --destdir) + shift + destdir="$1" + ;; + --version) + shift + version="$1" + ;; + --static) + static=1 + ;; + --tags) + shift + tags="$1" + ;; + --) + break + shift + ;; + -?*) + echo "Unknown option: ${1}. See --help." >&2 + exit 2 + ;; + *) + break + esac + shift +done + +configdir="${destdir}etc/maddy" + +if [ "$version" = "" ]; then + version=unknown + if [ -e .version ]; then + version="$(cat .version)" + fi + if [ -e .git ] && command -v git 2>/dev/null >/dev/null; then + version="${version}+$(git rev-parse --short HEAD)" + fi +fi + +set -e + +build_man_pages() { + set +e + if ! command -v scdoc >/dev/null 2>/dev/null; then + echo '-- [!] No scdoc utility found. Skipping man pages building.' >&2 + set -e + return + fi + set -e + + echo '-- Building man pages...' >&2 + + mkdir -p "${builddir}/man" + for f in ./docs/man/*.1.scd; do + scdoc < "$f" > "${builddir}/man/$(basename "$f" .scd)" + done +} + +build() { + mkdir -p "${builddir}" + echo "-- Version: ${version}" >&2 + if [ "$(go env CC)" = "" ]; then + echo '-- [!] No C compiler available. maddy will be built without SQLite3 support and default configuration will be unusable.' >&2 + fi + + if [ "$static" -eq 1 ]; then + echo "-- Building main server executable..." >&2 + # This is literally impossible to specify this line of arguments as part of ${GOFLAGS} + # using only POSIX sh features (and even with Bash extensions I can't figure it out). + go build -trimpath -buildmode pie -tags "$tags osusergo netgo static_build" \ + -ldflags "-extldflags '-fno-PIC -static' -X \"github.com/foxcpp/maddy.Version=${version}\"" \ + -o "${builddir}/maddy" ${GOFLAGS} ./cmd/maddy + else + echo "-- Building main server executable..." >&2 + go build -tags "$tags" -trimpath -ldflags="-X \"github.com/foxcpp/maddy.Version=${version}\"" -o "${builddir}/maddy" ${GOFLAGS} ./cmd/maddy + fi + + build_man_pages + + echo "-- Copying misc files..." >&2 + + mkdir -p "${builddir}/systemd" + cp dist/systemd/*.service "${builddir}/systemd/" + cp maddy.conf "${builddir}/maddy.conf" +} + +install() { + echo "-- Installing built files..." >&2 + + command install -m 0755 -d "${destdir}/${prefix}/bin/" + command install -m 0755 "${builddir}/maddy" "${destdir}/${prefix}/bin/" + command ln -sf maddy "${destdir}/${prefix}/bin/maddyctl" + command install -m 0755 -d "${configdir}" + + + # We do not want to overwrite existing configuration. + # If the file exists, then save it with .default suffix and warn user. + if [ ! -e "${configdir}/maddy.conf" ]; then + command install -m 0644 ./maddy.conf "${configdir}/maddy.conf" + else + echo "-- [!] Configuration file ${configdir}/maddy.conf exists, saving to ${configdir}/maddy.conf.default" >&2 + command install -m 0644 ./maddy.conf "${configdir}/maddy.conf.default" + fi + + # Attempt to install systemd units only for Linux. + # Check is done using GOOS instead of uname -s to account for possible + # package cross-compilation. + # Though go command might be unavailable if build.sh is run + # with sudo and go installation is user-specific, so fallback + # to using uname -s in the end. + set +e + if command -v go >/dev/null 2>/dev/null; then + set -e + if [ "$(go env GOOS)" = "linux" ]; then + command install -m 0755 -d "${destdir}/${prefix}/lib/systemd/system/" + command install -m 0644 "${builddir}"/systemd/*.service "${destdir}/${prefix}/lib/systemd/system/" + fi + else + set -e + if [ "$(uname -s)" = "Linux" ]; then + command install -m 0755 -d "${destdir}/${prefix}/lib/systemd/system/" + command install -m 0644 "${builddir}"/systemd/*.service "${destdir}/${prefix}/lib/systemd/system/" + fi + fi + + if [ -e "${builddir}"/man ]; then + command install -m 0755 -d "${destdir}/${prefix}/share/man/man1/" + for f in "${builddir}"/man/*.1; do + command install -m 0644 "$f" "${destdir}/${prefix}/share/man/man1/" + done + fi +} + +# Old build.sh compatibility +install_pkg() { + echo "-- [!] Replace 'install_pkg' with 'install' in build.sh invocation" >&2 + install +} +package() { + echo "-- [!] Replace 'package' with 'build' in build.sh invocation" >&2 + build +} + +if [ $# -eq 0 ]; then + build +else + for arg in "$@"; do + eval "$arg" + done +fi diff --git a/cmd/README.md b/cmd/README.md new file mode 100644 index 0000000..3132fea --- /dev/null +++ b/cmd/README.md @@ -0,0 +1,11 @@ +maddy executables +------------------- + +### maddy + +Main server executable. + +### maddy-pam-helper, maddy-shadow-helper + +Utilities compatible with the auth.external module that call libpam or read +/etc/shadow on Unix systems. diff --git a/cmd/maddy-pam-helper/README.md b/cmd/maddy-pam-helper/README.md new file mode 100644 index 0000000..2bfb179 --- /dev/null +++ b/cmd/maddy-pam-helper/README.md @@ -0,0 +1,65 @@ +## maddy-pam-helper + +External setuid binary for interaction with shadow passwords database or other +privileged objects necessary to run PAM authentication. + +### Building + +It is really easy to build it using any GCC: +``` +gcc pam.c main.c -lpam -o maddy-pam-helper +``` + +Yes, it is not a Go binary. + + +### Installation + +maddy-pam-helper is kinda dangerous binary and should not be allowed to be +executed by everybody but maddy's user. At the same moment it needs to have +access to read-protected files. For this reason installation should be done +very carefully to make sure to not introduce any security "holes". + +#### First method + +```shell +chown maddy: /usr/bin/maddy-pam-helper +chmod u+x,g-x,o-x /usr/bin/maddy-pam-helper +``` + +Also maddy-pam-helper needs access to /etc/shadow, one of the ways to provide +it is to set file capability CAP_DAC_READ_SEARCH: +``` +setcap cap_dac_read_search+ep /usr/bin/maddy-pam-helper +``` + +#### Second method + +Another, less restrictive is to make it setuid-root (assuming you have both maddy user and group): +``` +chown root:maddy /usr/bin/maddy-pam-helper +chmod u+xs,g+x,o-x /usr/bin/maddy-pam-helper +``` + +#### Third method + +The best way actually is to create `shadow` group and grant access to +/etc/shadow to it and then make maddy-pam-helper setgid-shadow: +``` +groupadd shadow +chown :shadow /etc/shadow +chmod g+r /etc/shadow +chown maddy:shadow /usr/bin/maddy-pam-helper +chmod u+x,g+xs /usr/bin/maddy-pam-helper +``` + +Pick what works best for you. + +### PAM service + +maddy-pam-helper uses custom service instead of pretending to be su or sudo. +Because of this you should configure PAM to accept it. + +Minimal example using local passwd/shadow database for authentication can be +found in [maddy.conf][maddy.conf] file. +It should be put into /etc/pam.d/maddy. diff --git a/cmd/maddy-pam-helper/maddy.conf b/cmd/maddy-pam-helper/maddy.conf new file mode 100644 index 0000000..c5bef88 --- /dev/null +++ b/cmd/maddy-pam-helper/maddy.conf @@ -0,0 +1,3 @@ +#%PAM-1.0 +auth required pam_unix.so +account required pam_unix.so diff --git a/cmd/maddy-pam-helper/main.c b/cmd/maddy-pam-helper/main.c new file mode 100644 index 0000000..aec687d --- /dev/null +++ b/cmd/maddy-pam-helper/main.c @@ -0,0 +1,51 @@ +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include "pam.h" + +/* +I really doubt it is a good idea to bring Go to the binary whose primary task +is to call libpam using CGo anyway. +*/ + +int run(void) { + char *username = NULL, *password = NULL; + size_t username_buf_len = 0, password_buf_len = 0; + + ssize_t username_len = getline(&username, &username_buf_len, stdin); + if (username_len < 0) { + perror("getline username"); + return 2; + } + + ssize_t password_len = getline(&password, &password_buf_len, stdin); + if (password_len < 0) { + perror("getline password"); + return 2; + } + + // Cut trailing \n. + if (username_len > 0) { + username[username_len - 1] = 0; + } + if (password_len > 0) { + password[password_len - 1] = 0; + } + + struct error_obj err = run_pam_auth(username, password); + if (err.status != 0) { + if (err.status == 2) { + fprintf(stderr, "%s: %s\n", err.func_name, err.error_msg); + } + return err.status; + } + + return 0; +} + +#ifndef CGO +int main() { + return run(); +} +#endif diff --git a/cmd/maddy-pam-helper/main.go b/cmd/maddy-pam-helper/main.go new file mode 100644 index 0000000..0a3879c --- /dev/null +++ b/cmd/maddy-pam-helper/main.go @@ -0,0 +1,38 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package main + +/* +#cgo LDFLAGS: -lpam +#cgo CFLAGS: -DCGO -Wall -Wextra -Werror -Wno-unused-parameter -Wno-error=unused-parameter -Wpedantic -std=c99 +extern int run(); +*/ +import "C" +import "os" + +/* +Apparently, some people would not want to build it manually by calling GCC. +Here we do it for them. Not going to tell them that resulting file is 800KiB +bigger than one built using only C compiler. +*/ + +func main() { + i := int(C.run()) + os.Exit(i) +} diff --git a/cmd/maddy-pam-helper/pam.c b/cmd/maddy-pam-helper/pam.c new file mode 100644 index 0000000..4829164 --- /dev/null +++ b/cmd/maddy-pam-helper/pam.c @@ -0,0 +1,100 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2022 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include "pam.h" + +static int conv_func(int num_msg, const struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) { + struct pam_response *reply = malloc(sizeof(struct pam_response)); + if (reply == NULL) { + return PAM_CONV_ERR; + } + + char* password_cpy = malloc(strlen((char*)appdata_ptr)+1); + if (password_cpy == NULL) { + return PAM_CONV_ERR; + } + memcpy(password_cpy, (char*)appdata_ptr, strlen((char*)appdata_ptr)+1); + + reply->resp = password_cpy; + reply->resp_retcode = 0; + + // PAM frees pam_response for us. + *resp = reply; + + return PAM_SUCCESS; +} + +struct error_obj run_pam_auth(const char *username, char *password) { + const struct pam_conv local_conv = { conv_func, password }; + pam_handle_t *local_auth = NULL; + int status = pam_start("maddy", username, &local_conv, &local_auth); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + ret_val.status = 2; + ret_val.func_name = "pam_start"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + status = pam_authenticate(local_auth, PAM_SILENT|PAM_DISALLOW_NULL_AUTHTOK); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + if (status == PAM_AUTH_ERR || status == PAM_USER_UNKNOWN) { + ret_val.status = 1; + } else { + ret_val.status = 2; + } + ret_val.func_name = "pam_authenticate"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + status = pam_acct_mgmt(local_auth, PAM_SILENT|PAM_DISALLOW_NULL_AUTHTOK); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + if (status == PAM_AUTH_ERR || status == PAM_USER_UNKNOWN || status == PAM_NEW_AUTHTOK_REQD) { + ret_val.status = 1; + } else { + ret_val.status = 2; + } + ret_val.func_name = "pam_acct_mgmt"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + status = pam_end(local_auth, status); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + ret_val.status = 2; + ret_val.func_name = "pam_end"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + struct error_obj ret_val; + ret_val.status = 0; + ret_val.func_name = NULL; + ret_val.error_msg = NULL; + return ret_val; +} + diff --git a/cmd/maddy-pam-helper/pam.h b/cmd/maddy-pam-helper/pam.h new file mode 100644 index 0000000..e9831ec --- /dev/null +++ b/cmd/maddy-pam-helper/pam.h @@ -0,0 +1,27 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +#pragma once + +struct error_obj { + int status; + const char* func_name; + const char* error_msg; +}; + +struct error_obj run_pam_auth(const char *username, char *password); diff --git a/cmd/maddy-shadow-helper/README.md b/cmd/maddy-shadow-helper/README.md new file mode 100644 index 0000000..2202420 --- /dev/null +++ b/cmd/maddy-shadow-helper/README.md @@ -0,0 +1,47 @@ +## maddy-shadow-helper + +External helper binary for interaction with shadow passwords database. +Unlike maddy-pam-helper it supports only local shadow database but it does +not have any C dependencies. + +### Installation + +maddy-shadow-helper is kinda dangerous binary and should not be allowed to be +executed by everybody but maddy's user. At the same moment it needs to have +access to read-protected files. For this reason installation should be done +very carefully to make sure to not introduce any security "holes". + +#### First method + +```shell +chown maddy: /usr/bin/maddy-shadow-helper +chmod u+x,g-x,o-x /usr/bin/maddy-shadow-helper +``` + +Also maddy-shadow-helper needs access to /etc/shadow, one of the ways to provide +it is to set file capability CAP_DAC_READ_SEARCH: +``` +setcap cap_dac_read_search+ep /usr/bin/maddy-shadow-helper +``` + +#### Second method + +Another, less restrictive is to make it setuid-root (assuming you have both maddy user and group): +``` +chown root:maddy /usr/bin/maddy-shadow-helper +chmod u+xs,g+x,o-x /usr/bin/maddy-shadow-helper +``` + +#### Third method + +The best way actually is to create `shadow` group and grant access to +/etc/shadow to it and then make maddy-shadow-helper setgid-shadow: +``` +groupadd shadow +chown :shadow /etc/shadow +chmod g+r /etc/shadow +chown maddy:shadow /usr/bin/maddy-shadow-helper +chmod u+x,g+xs /usr/bin/maddy-shadow-helper +``` + +Pick what works best for you. diff --git a/cmd/maddy-shadow-helper/main.go b/cmd/maddy-shadow-helper/main.go new file mode 100644 index 0000000..0c6bcd5 --- /dev/null +++ b/cmd/maddy-shadow-helper/main.go @@ -0,0 +1,71 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package main + +import ( + "bufio" + "errors" + "fmt" + "os" + + "github.com/foxcpp/maddy/internal/auth/shadow" +) + +func main() { + scnr := bufio.NewScanner(os.Stdin) + + if !scnr.Scan() { + fmt.Fprintln(os.Stderr, scnr.Err()) + os.Exit(2) + } + username := scnr.Text() + + if !scnr.Scan() { + fmt.Fprintln(os.Stderr, scnr.Err()) + os.Exit(2) + } + password := scnr.Text() + + ent, err := shadow.Lookup(username) + if err != nil { + if errors.Is(err, shadow.ErrNoSuchUser) { + os.Exit(1) + } + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + + if !ent.IsAccountValid() { + fmt.Fprintln(os.Stderr, "account is expired") + os.Exit(1) + } + + if !ent.IsPasswordValid() { + fmt.Fprintln(os.Stderr, "password is expired") + os.Exit(1) + } + + if err := ent.VerifyPassword(password); err != nil { + if errors.Is(err, shadow.ErrWrongPassword) { + os.Exit(1) + } + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/maddy/main.go b/cmd/maddy/main.go new file mode 100644 index 0000000..1007417 --- /dev/null +++ b/cmd/maddy/main.go @@ -0,0 +1,29 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package main + +import ( + _ "github.com/foxcpp/maddy" + maddycli "github.com/foxcpp/maddy/internal/cli" + _ "github.com/foxcpp/maddy/internal/cli/ctl" +) + +func main() { + maddycli.Run() +} diff --git a/config.go b/config.go new file mode 100644 index 0000000..2f57b47 --- /dev/null +++ b/config.go @@ -0,0 +1,120 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package maddy + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" +) + +/* +Config matchers for module interfaces. +*/ + +// logOut structure wraps log.Output and preserves +// configuration directive it was constructed from, allowing +// dynamic reinitialization for purposes of log file rotation. +type logOut struct { + args []string + log.Output +} + +func logOutput(_ *config.Map, node config.Node) (interface{}, error) { + if len(node.Args) == 0 { + return nil, config.NodeErr(node, "expected at least 1 argument") + } + if len(node.Children) != 0 { + return nil, config.NodeErr(node, "can't declare block here") + } + + return LogOutputOption(node.Args) +} + +func LogOutputOption(args []string) (log.Output, error) { + outs := make([]log.Output, 0, len(args)) + for i, arg := range args { + switch arg { + case "stderr": + outs = append(outs, log.WriterOutput(os.Stderr, false)) + case "stderr_ts": + outs = append(outs, log.WriterOutput(os.Stderr, true)) + case "syslog": + syslogOut, err := log.SyslogOutput() + if err != nil { + return nil, fmt.Errorf("failed to connect to syslog daemon: %v", err) + } + outs = append(outs, syslogOut) + case "off": + if len(args) != 1 { + return nil, errors.New("'off' can't be combined with other log targets") + } + return log.NopOutput{}, nil + default: + // Log file paths are converted to absolute to make sure + // we will be able to recreate them in right location + // after changing working directory to the state dir. + absPath, err := filepath.Abs(arg) + if err != nil { + return nil, err + } + // We change the actual argument, so logOut object will + // keep the absolute path for reinitialization. + args[i] = absPath + + w, err := os.OpenFile(absPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o666) + if err != nil { + return nil, fmt.Errorf("failed to create log file: %v", err) + } + + outs = append(outs, log.WriteCloserOutput(w, true)) + } + } + + if len(outs) == 1 { + return logOut{args, outs[0]}, nil + } + return logOut{args, log.MultiOutput(outs...)}, nil +} + +func defaultLogOutput() (interface{}, error) { + return log.DefaultLogger.Out, nil +} + +func reinitLogging() { + out, ok := log.DefaultLogger.Out.(logOut) + if !ok { + log.Println("Can't reinitialize logger because it was replaced before, this is a bug") + return + } + + newOut, err := LogOutputOption(out.args) + if err != nil { + log.Println("Can't reinitialize logger:", err) + return + } + + out.Close() + + log.DefaultLogger.Out = newOut +} diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000..990fced --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,6 @@ +# Community contributed resources + +Disclaimer: Nothing inside subdirectories here is directly supported by Maddy +Mail Server maintainers. Some community members may be able to help you or not. + +- Kubernetes helm chart is maintained by @acim. diff --git a/contrib/kubernetes/chart/.helmignore b/contrib/kubernetes/chart/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/contrib/kubernetes/chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/contrib/kubernetes/chart/Chart.yaml b/contrib/kubernetes/chart/Chart.yaml new file mode 100644 index 0000000..85468f8 --- /dev/null +++ b/contrib/kubernetes/chart/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: maddy +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.2.6 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 0.4.0 diff --git a/contrib/kubernetes/chart/README.md b/contrib/kubernetes/chart/README.md new file mode 100644 index 0000000..14e2e74 --- /dev/null +++ b/contrib/kubernetes/chart/README.md @@ -0,0 +1,69 @@ +# maddy Helm chart for Kubernetes + +![Version: 0.2.5](https://img.shields.io/badge/Version-0.2.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.4.1](https://img.shields.io/badge/AppVersion-0.4.1-informational?style=flat-square) + +This is just initial effort to run maddy within Kubernetes cluster. We have used Deployment resource which has some downsides +but at least this chart will allow you to install maddy relatively easily on your Kubernetes cluster. We have considered +StatefulSet and DaemonSet but such solutions would require much more configuration and in casae of DaemonSet also a TCP +load balancer in front of the nodes. + +## Requirement + +In order to run maddy properly, you need to have TLS secret under name maddy present in the cluster. If you have commercial +certificate, you can create it by the following command: + +```sh +kubectl create secret tls maddy --cert=fullchain.pem --key=privkey.pem +``` + +If you use cert-manager, just create the secret under name maddy. + +## Replication + +Default for this chart is 1 replica of maddy. If you try to increase this, you will probably get an error because of +the busy ports 25, 143, 587, etc. We do not support this feature at the moment, so please use just 1 replica. Like said +at the beginning of this document, multiple replicas would probably require to switch do DaemonSet which would further require +to have TCP load balancer and shared storage between all replicas. This is not supported by this chart, sorry. +This chart is used on one node cluster and then installation is straight forward, like described bellow, but if you have +multiple node cluster, please use taints and tolerations to select the desired node. This chart supports tolerations to +be set. + +## Configuration + +| Key | Type | Default | Description | +| -------------------------- | ------ | ----------------- | ----------- | +| affinity | object | `{}` | | +| fullnameOverride | string | `""` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"foxcpp/maddy"` | | +| image.tag | string | `""` | | +| imagePullSecrets | list | `[]` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| persistence.accessMode | string | `"ReadWriteOnce"` | | +| persistence.annotations | object | `{}` | | +| persistence.enabled | bool | `false` | | +| persistence.path | string | `"/data"` | | +| persistence.size | string | `"128Mi"` | | +| podAnnotations | object | `{}` | | +| podSecurityContext | object | `{}` | | +| replicaCount | int | `1` | | +| resources | object | `{}` | | +| securityContext | object | `{}` | | +| service.type | string | `"NodePort"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `""` | | +| tolerations | list | `[]` | | + +## Installing the chart + +```sh +helm upgrade --install maddy ./chart --set service.externapIPs[0]=1.2.3.4 +``` + +1.2.3.4 is your public IP of the node. + +## maddy configuration + +Feel free to tweak files/maddy.conf and files/aliases according to your needs. diff --git a/contrib/kubernetes/chart/files/aliases b/contrib/kubernetes/chart/files/aliases new file mode 100644 index 0000000..a486390 --- /dev/null +++ b/contrib/kubernetes/chart/files/aliases @@ -0,0 +1 @@ +info@example.org: foxcpp@example.org diff --git a/contrib/kubernetes/chart/files/maddy.conf b/contrib/kubernetes/chart/files/maddy.conf new file mode 100644 index 0000000..6e8b66c --- /dev/null +++ b/contrib/kubernetes/chart/files/maddy.conf @@ -0,0 +1,171 @@ +## maddy 0.3 - default configuration file (2020-05-31) +# Suitable for small-scale deployments. Uses its own format for local users DB, +# should be managed via maddy subcommands. +# +# See tutorials at https://foxcpp.dev/maddy for guidance on typical +# configuration changes. +# +# See manual pages (also available at https://foxcpp.dev/maddy) for reference +# documentation. + +# ---------------------------------------------------------------------------- +# Base variables + +$(hostname) = mx1.example.org +$(primary_domain) = example.org +$(local_domains) = $(primary_domain) + +tls file /etc/maddy/certs/fullchain.pem /etc/maddy/certs/privkey.pem + +# ---------------------------------------------------------------------------- +# Local storage & authentication + +# pass_table provides local hashed passwords storage for authentication of +# users. It can be configured to use any "table" module, in default +# configuration a table in SQLite DB is used. +# Table can be replaced to use e.g. a file for passwords. Or pass_table module +# can be replaced altogether to use some external source of credentials (e.g. +# PAM, /etc/shadow file). +# +# If table module supports it (sql_table does) - credentials can be managed +# using 'maddy creds' command. + +auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } +} + +# imapsql module stores all indexes and metadata necessary for IMAP using a +# relational database. It is used by IMAP endpoint for mailbox access and +# also by SMTP & Submission endpoints for delivery of local messages. +# +# IMAP accounts, mailboxes and all message metadata can be inspected using +# imap-* subcommands of maddy. + +storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db +} + +# ---------------------------------------------------------------------------- +# SMTP endpoints + message routing + +hostname $(hostname) + +msgpipeline local_routing { + dmarc yes + check { + require_matching_ehlo + require_mx_record + dkim + spf + } + + # Insert handling for special-purpose local domains here. + # e.g. + # destination lists.example.org { + # deliver_to lmtp tcp://127.0.0.1:8024 + # } + + destination postmaster $(local_domains) { + modify { + replace_rcpt regexp "(.+)\+(.+)@(.+)" "$1@$3" + replace_rcpt file /data/aliases + } + + deliver_to &local_mailboxes + } + + default_destination { + reject 550 5.1.1 "User doesn't exist" + } +} + +smtp tcp://0.0.0.0:25 { + limits { + # Up to 20 msgs/sec across max. 10 SMTP connections. + all rate 20 1s + all concurrency 10 + } + + source $(local_domains) { + reject 501 5.1.8 "Use Submission for outgoing SMTP" + } + default_source { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + reject 550 5.1.1 "User doesn't exist" + } + } +} + +submission tls://0.0.0.0:465 tcp://0.0.0.0:587 { + limits { + # Up to 50 msgs/sec across any amount of SMTP connections. + all rate 50 1s + } + + auth &local_authdb + + source $(local_domains) { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + modify { + dkim $(primary_domain) $(local_domains) default + } + deliver_to &remote_queue + } + } + default_source { + reject 501 5.1.8 "Non-local sender domain" + } +} + +target.remote outbound_delivery { + limits { + # Up to 20 msgs/sec across max. 10 SMTP connections + # for each recipient domain. + destination rate 20 1s + destination concurrency 10 + } + mx_auth { + dane + mtasts { + cache fs + fs_dir mtasts_cache/ + } + local_policy { + min_tls_level encrypted + min_mx_level none + } + } +} + +target.queue remote_queue { + target &outbound_delivery + + autogenerated_msg_domain $(primary_domain) + bounce { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + reject 550 5.0.0 "Refusing to send DSNs to non-local addresses" + } + } +} + +# ---------------------------------------------------------------------------- +# IMAP endpoints + +imap tls://0.0.0.0:993 tcp://0.0.0.0:143 { + auth &local_authdb + storage &local_mailboxes +} diff --git a/contrib/kubernetes/chart/templates/NOTES.txt b/contrib/kubernetes/chart/templates/NOTES.txt new file mode 100644 index 0000000..e69de29 diff --git a/contrib/kubernetes/chart/templates/_helpers.tpl b/contrib/kubernetes/chart/templates/_helpers.tpl new file mode 100644 index 0000000..99b3e0e --- /dev/null +++ b/contrib/kubernetes/chart/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "maddy.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "maddy.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "maddy.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "maddy.labels" -}} +helm.sh/chart: {{ include "maddy.chart" . }} +{{ include "maddy.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "maddy.selectorLabels" -}} +app.kubernetes.io/name: {{ include "maddy.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "maddy.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "maddy.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/contrib/kubernetes/chart/templates/configmap.yaml b/contrib/kubernetes/chart/templates/configmap.yaml new file mode 100644 index 0000000..5d24a25 --- /dev/null +++ b/contrib/kubernetes/chart/templates/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{include "maddy.fullname" .}} + labels: {{- include "maddy.labels" . | nindent 4}} +data: + maddy.conf: | +{{ tpl (.Files.Get "files/maddy.conf") . | printf "%s" | indent 4 }} + aliases: | +{{ tpl (.Files.Get "files/aliases") . | printf "%s" | indent 4 }} diff --git a/contrib/kubernetes/chart/templates/deployment.yaml b/contrib/kubernetes/chart/templates/deployment.yaml new file mode 100644 index 0000000..f0f1492 --- /dev/null +++ b/contrib/kubernetes/chart/templates/deployment.yaml @@ -0,0 +1,113 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "maddy.fullname" . }} + labels: + {{- include "maddy.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "maddy.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ tpl (.Files.Get "files/maddy.conf") . | printf "%s" | sha256sum }} + checksum/aliases: {{ tpl (.Files.Get "files/aliases") . | printf "%s" | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "maddy.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "maddy.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + - name: init + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: busybox + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - cp /tmp/maddy/* /data/. + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: {{ .Values.persistence.path }} + {{- if .Values.persistence.subPath }} + subPath: {{ .Values.persistence.subPath }} + {{- end }} + - name: config + mountPath: /tmp/maddy + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: smtp + containerPort: 25 + protocol: TCP + - name: imaps + containerPort: 993 + protocol: TCP + - name: starttls + containerPort: 587 + protocol: TCP + # livenessProbe: + # httpGet: + # path: / + # port: http + # readinessProbe: + # httpGet: + # path: / + # port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: {{ .Values.persistence.path }} + {{- if .Values.persistence.subPath }} + subPath: {{ .Values.persistence.subPath }} + {{- end }} + - name: tls + mountPath: /etc/maddy/certs/fullchain.pem + subPath: tls.crt + - name: tls + mountPath: /etc/maddy/certs/privkey.pem + subPath: tls.key + volumes: + - name: data + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ default (include "maddy.fullname" .) .Values.persistence.existingClaim }} + {{- else }} + emptyDir: {} + {{- end }} + - name: config + configMap: + name: {{include "maddy.fullname" .}} + - name: tls + secret: + secretName: {{include "maddy.fullname" .}} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/contrib/kubernetes/chart/templates/pvc.yaml b/contrib/kubernetes/chart/templates/pvc.yaml new file mode 100644 index 0000000..a9ffe7f --- /dev/null +++ b/contrib/kubernetes/chart/templates/pvc.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "maddy.fullname" . }} + annotations: + {{- with .Values.persistence.annotations }} + {{ toYaml . | indent 4 }} + {{- end }} + labels: + {{- include "maddy.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.accessMode }} + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass }} + {{- end }} +{{- end -}} + diff --git a/contrib/kubernetes/chart/templates/service.yaml b/contrib/kubernetes/chart/templates/service.yaml new file mode 100644 index 0000000..7320bd3 --- /dev/null +++ b/contrib/kubernetes/chart/templates/service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "maddy.fullname" . }} + labels: + {{- include "maddy.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: 25 + targetPort: smtp + protocol: TCP + name: smtp + - port: 993 + targetPort: imaps + protocol: TCP + name: imaps + - port: 587 + targetPort: starttls + protocol: TCP + name: starttls + selector: + {{- include "maddy.selectorLabels" . | nindent 4 }} + {{- with .Values.service.externalIPs }} + externalIPs: + {{- toYaml . | nindent 6 }} + {{- end -}} diff --git a/contrib/kubernetes/chart/templates/serviceaccount.yaml b/contrib/kubernetes/chart/templates/serviceaccount.yaml new file mode 100644 index 0000000..ace0978 --- /dev/null +++ b/contrib/kubernetes/chart/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "maddy.serviceAccountName" . }} + labels: + {{- include "maddy.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/contrib/kubernetes/chart/templates/tests/test-connection.yaml b/contrib/kubernetes/chart/templates/tests/test-connection.yaml new file mode 100644 index 0000000..bb163b3 --- /dev/null +++ b/contrib/kubernetes/chart/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "maddy.fullname" . }}-test-connection" + labels: + {{- include "maddy.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "maddy.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/contrib/kubernetes/chart/values.yaml b/contrib/kubernetes/chart/values.yaml new file mode 100644 index 0000000..ede78ca --- /dev/null +++ b/contrib/kubernetes/chart/values.yaml @@ -0,0 +1,74 @@ +# Default values for maddy. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 # Multiple replicas are not supported, please don't change this. + +image: + repository: foxcpp/maddy + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: + {} + # fsGroup: 2000 + +securityContext: + {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +# Set externalPIs to your public IP(s) of the node running maddy. In case of multiple nodes, you need to set tolerations +# and taints in order to run maddy on the exact node. +service: + type: NodePort + # externalIPs: + +resources: + {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +persistence: + enabled: false + # existingClaim: "" + accessMode: ReadWriteOnce + size: 128Mi + # storageClass: "" + path: /data + annotations: {} + # subPath: "" # only mount a subpath of the Volume into the pod + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/directories.go b/directories.go new file mode 100644 index 0000000..b3930db --- /dev/null +++ b/directories.go @@ -0,0 +1,46 @@ +//go:build !docker +// +build !docker + +package maddy + +var ( + // ConfigDirectory specifies platform-specific value + // that should be used as a location of default configuration + // + // It should not be changed and is defined as a variable + // only for purposes of modification using -X linker flag. + ConfigDirectory = "/etc/maddy" + + // DefaultStateDirectory specifies platform-specific + // default for StateDirectory. + // + // Most code should use StateDirectory instead since + // it will contain the effective location of the state + // directory. + // + // It should not be changed and is defined as a variable + // only for purposes of modification using -X linker flag. + DefaultStateDirectory = "/var/lib/maddy" + + // DefaultRuntimeDirectory specifies platform-specific + // default for RuntimeDirectory. + // + // Most code should use RuntimeDirectory instead since + // it will contain the effective location of the state + // directory. + // + // It should not be changed and is defined as a variable + // only for purposes of modification using -X linker flag. + DefaultRuntimeDirectory = "/run/maddy" + + // DefaultLibexecDirectory specifies platform-specific + // default for LibexecDirectory. + // + // Most code should use LibexecDirectory since it will + // contain the effective location of the libexec + // directory. + // + // It should not be changed and is defined as a variable + // only for purposes of modification using -X linker flag. + DefaultLibexecDirectory = "/usr/lib/maddy" +) diff --git a/directories_docker.go b/directories_docker.go new file mode 100644 index 0000000..4869f60 --- /dev/null +++ b/directories_docker.go @@ -0,0 +1,11 @@ +//go:build docker +// +build docker + +package maddy + +var ( + ConfigDirectory = "/data" + DefaultStateDirectory = "/data" + DefaultRuntimeDirectory = "/tmp" + DefaultLibexecDirectory = "/usr/lib/maddy" +) diff --git a/dist/README.md b/dist/README.md new file mode 100644 index 0000000..d057f0e --- /dev/null +++ b/dist/README.md @@ -0,0 +1,41 @@ +Distribution files for maddy +------------------------------ + +**Disclaimer:** Most of the files here are maintained in a "best-effort" way. +That is, they may break or become outdated from time to time. Caveat emptor. + +## integration + scripts + +These directories provide pre-made configuration snippets suitable for +easy integration with external software. + +Usually, this is what you use when you put `import integration/something` in +your config. + +## systemd unit + +`maddy.service` launches using default config path (/etc/maddy/maddy.conf). +`maddy@.service` launches maddy using custom config path. E.g. +`maddy@foo.service` will use /etc/maddy/foo.conf. + +Additionally, unit files apply strict sandboxing, limiting maddy permissions on +the system to a bare minimum. Subset of these options makes it impossible for +privileged authentication helper binaries to gain required permissions, so you +may have to disable it when using system account-based authentication with +maddy running as a unprivileged user. + +## fail2ban configuration + +Configuration files for use with fail2ban. Assume either `backend = systemd` specified +in system-wide configuration or log file written to /var/log/maddy/maddy.log. + +See https://github.com/foxcpp/maddy/wiki/fail2ban-configuration for details. + +## logrotate configuration + +Meant for logs rotation when logging to file is used. + +## vim ftdetect/ftplugin/syntax files + +Minimal supplement to make configuration files more readable and help you see +typos in directive names. diff --git a/dist/apparmor/dev.foxcpp.maddy b/dist/apparmor/dev.foxcpp.maddy new file mode 100644 index 0000000..9486c76 --- /dev/null +++ b/dist/apparmor/dev.foxcpp.maddy @@ -0,0 +1,38 @@ +# AppArmor profile for maddy daemon. +# vim:syntax=apparmor:ts=2:sw=2:et + +#include + +profile dev.foxcpp.maddy /usr{/local,}/bin/maddy { + #include + #include + #include + /etc/ca-certificates/** r, + + /etc/resolv.conf r, + /proc/sys/net/core/somaxconn r, + /sys/kernel/mm/transparent_hugepage/hpage_pmd_size r, + deny ptrace, + capability net_bind_service, + network tcp, + network unix, + + # systemd process management and Type=notify + signal (receive) peer=unconfined, + signal (receive) peer=/usr/bin/systemd, + unix (create, connect, send, setopt) type=dgram addr=@*, + /run/systemd/notify w, + + /etc/maddy/** r, + owner /run/maddy/ rw, + owner /run/maddy/** rwkl, + owner /var/lib/maddy/ rw, + owner /var/lib/maddy/** rwk, + owner /var/lib/maddy/**.db-{wal,shm} rmk, + + /usr{/local,}/lib/maddy/* PUx, + + /usr{/local,}/bin/maddy{,ctl} rmix, + + #include if exists +} diff --git a/dist/fail2ban/filter.d/maddy-auth.conf b/dist/fail2ban/filter.d/maddy-auth.conf new file mode 100644 index 0000000..bbaf31b --- /dev/null +++ b/dist/fail2ban/filter.d/maddy-auth.conf @@ -0,0 +1,6 @@ +[INCLUDES] +before = common.conf + +[Definition] +failregex = authentication failed\t\{\"reason\":\".*\",\"src_ip\"\:\":\d+\"\,\"username\"\:\".*\"\}$ +journalmatch = _SYSTEMD_UNIT=maddy.service + _COMM=maddy diff --git a/dist/fail2ban/filter.d/maddy-dictonary-attack.conf b/dist/fail2ban/filter.d/maddy-dictonary-attack.conf new file mode 100644 index 0000000..1b233fa --- /dev/null +++ b/dist/fail2ban/filter.d/maddy-dictonary-attack.conf @@ -0,0 +1,7 @@ +[INCLUDES] +before = common.conf + +[Definition] +failregex = smtp\: MAIL FROM error repeated a lot\, possible dictonary attack\t\{\"count\"\:\d+,\"msg_id\":\".+\",\"src_ip\"\:\":\d+\"\}$ + smtp\: too many RCPT errors\, possible dictonary attack\t\{\"msg_id\":\".+\","src_ip":":\d+\"\} +journalmatch = _SYSTEMD_UNIT=maddy.service + _COMM=maddy diff --git a/dist/fail2ban/jail.d/maddy-auth.conf b/dist/fail2ban/jail.d/maddy-auth.conf new file mode 100644 index 0000000..1543246 --- /dev/null +++ b/dist/fail2ban/jail.d/maddy-auth.conf @@ -0,0 +1,5 @@ +[maddy-auth] +port = 993,465,25 +filter = maddy-auth +bantime = 96h +backend = systemd diff --git a/dist/fail2ban/jail.d/maddy-dictonary-attack.conf b/dist/fail2ban/jail.d/maddy-dictonary-attack.conf new file mode 100644 index 0000000..ebeb33f --- /dev/null +++ b/dist/fail2ban/jail.d/maddy-dictonary-attack.conf @@ -0,0 +1,7 @@ +[maddy-dictonary-attack] +port = 993,465,25 +filter = maddy-dictonary-attack +bantime = 72h +maxretry = 3 +findtime = 6h +backend = systemd diff --git a/dist/install.sh b/dist/install.sh new file mode 100755 index 0000000..33cb693 --- /dev/null +++ b/dist/install.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +DESTDIR=$DESTDIR +if [ -z "$PREFIX" ]; then + PREFIX=/usr/local +fi +if [ -z "$FAIL2BANDIR" ]; then + FAIL2BANDIR=/etc/fail2ban +fi +if [ -z "$CONFDIR" ]; then + CONFDIR=/etc/maddy +fi + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $script_dir + +install -Dm 0644 -t "$DESTDIR/$PREFIX/share/vim/vimfiles/ftdetect/" vim/ftdetect/maddy-conf.vim +install -Dm 0644 -t "$DESTDIR/$PREFIX/share/vim/vimfiles/ftplugin/" vim/ftplugin/maddy-conf.vim +install -Dm 0644 -t "$DESTDIR/$PREFIX/share/vim/vimfiles/syntax/" vim/syntax/maddy-conf.vim + +install -Dm 0644 -t "$DESTDIR/$FAIL2BANDIR/jail.d/" fail2ban/jail.d/* +install -Dm 0644 -t "$DESTDIR/$FAIL2BANDIR/filter.d/" fail2ban/filter.d/* + +install -Dm 0644 -t "$DESTDIR/$PREFIX/lib/systemd/system/" systemd/maddy.service systemd/maddy@.service diff --git a/dist/logrotate.d/maddy b/dist/logrotate.d/maddy new file mode 100644 index 0000000..2c264e7 --- /dev/null +++ b/dist/logrotate.d/maddy @@ -0,0 +1,7 @@ +/var/log/maddy/maddy.log { + missingok + su maddy maddy + postrotate + /usr/bin/killall -USR1 maddy + endscript +} diff --git a/dist/systemd/maddy.service b/dist/systemd/maddy.service new file mode 100644 index 0000000..ec1ac29 --- /dev/null +++ b/dist/systemd/maddy.service @@ -0,0 +1,82 @@ +[Unit] +Description=maddy mail server +Documentation=man:maddy(1) +Documentation=man:maddy.conf(5) +Documentation=https://maddy.email +After=network-online.target + +[Service] +Type=notify +NotifyAccess=main + +User=maddy +Group=maddy + +# cd to state directory to make sure any relative paths +# in config will be relative to it unless handled specially. +WorkingDirectory=/var/lib/maddy + +ConfigurationDirectory=maddy +RuntimeDirectory=maddy +StateDirectory=maddy +LogsDirectory=maddy +ReadOnlyPaths=/usr/lib/maddy +ReadWritePaths=/var/lib/maddy + +# Strict sandboxing. You have no reason to trust code written by strangers from GitHub. +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +ProtectKernelTunables=true +ProtectHostname=true +ProtectClock=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 + +# Additional sandboxing. You need to disable all of these options +# for privileged helper binaries (for system auth) to work correctly. +NoNewPrivileges=true +PrivateDevices=true +DeviceAllow=/dev/syslog +RestrictSUIDSGID=true +ProtectKernelModules=true +MemoryDenyWriteExecute=true +RestrictNamespaces=true +RestrictRealtime=true +LockPersonality=true + +# Graceful shutdown with a reasonable timeout. +TimeoutStopSec=7s +KillMode=mixed +KillSignal=SIGTERM + +# Required to bind on ports lower than 1024. +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE + +# Force all files created by maddy to be only readable by it +# and maddy group. +UMask=0007 + +# Bump FD limitations. Even idle mail server can have a lot of FDs open (think +# of idle IMAP connections, especially ones abandoned on the other end and +# slowly timing out). +LimitNOFILE=131072 + +# Limit processes count to something reasonable to +# prevent resources exhausting due to big amounts of helper +# processes launched. +LimitNPROC=512 + +# Restart server on any problem. +Restart=on-failure +# ... Unless it is a configuration problem. +RestartPreventExitStatus=2 + +ExecStart=/usr/local/bin/maddy run + +ExecReload=/bin/kill -USR1 $MAINPID +ExecReload=/bin/kill -USR2 $MAINPID + +[Install] +WantedBy=multi-user.target diff --git a/dist/systemd/maddy@.service b/dist/systemd/maddy@.service new file mode 100644 index 0000000..ea60ff8 --- /dev/null +++ b/dist/systemd/maddy@.service @@ -0,0 +1,78 @@ +[Unit] +Description=maddy mail server (using %i.conf) +Documentation=man:maddy(1) +Documentation=man:maddy.conf(5) +Documentation=https://maddy.email +After=network-online.target + +[Service] +Type=notify +NotifyAccess=main + +User=maddy +Group=maddy + +ConfigurationDirectory=maddy +RuntimeDirectory=maddy +StateDirectory=maddy +LogsDirectory=maddy +ReadOnlyPaths=/usr/lib/maddy +ReadWritePaths=/var/lib/maddy + +# Strict sandboxing. You have no reason to trust code written by strangers from GitHub. +PrivateTmp=true +PrivateHome=true +ProtectSystem=strict +ProtectKernelTunables=true +ProtectHostname=true +ProtectClock=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +DeviceAllow=/dev/syslog + +# Additional sandboxing. You need to disable all of these options +# for privileged helper binaries (for system auth) to work correctly. +NoNewPrivileges=true +PrivateDevices=true +RestrictSUIDSGID=true +ProtectKernelModules=true +MemoryDenyWriteExecute=true +RestrictNamespaces=true +RestrictRealtime=true +LockPersonality=true + +# Graceful shutdown with a reasonable timeout. +TimeoutStopSec=7s +KillMode=mixed +KillSignal=SIGTERM + +# Required to bind on ports lower than 1024. +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE + +# Force all files created by maddy to be only readable by it and +# maddy group. +UMask=0007 + +# Bump FD limitations. Even idle mail server can have a lot of FDs open (think +# of idle IMAP connections, especially ones abandoned on the other end and +# slowly timing out). +LimitNOFILE=131072 + +# Limit processes count to something reasonable to +# prevent resources exhausting due to big amounts of helper +# processes launched. +LimitNPROC=512 + +# Restart server on any problem. +Restart=on-failure +# ... Unless it is a configuration problem. +RestartPreventExitStatus=2 + +ExecStart=/usr/local/bin/maddy --config /etc/maddy/%i.conf run + +ExecReload=/bin/kill -USR1 $MAINPID +ExecReload=/bin/kill -USR2 $MAINPID + +[Install] +WantedBy=multi-user.target diff --git a/dist/vim/ftdetect/maddy-conf.vim b/dist/vim/ftdetect/maddy-conf.vim new file mode 100644 index 0000000..672b137 --- /dev/null +++ b/dist/vim/ftdetect/maddy-conf.vim @@ -0,0 +1 @@ +au BufNewFile,BufRead /etc/maddy/*,maddy.conf setf maddy-conf diff --git a/dist/vim/ftplugin/maddy-conf.vim b/dist/vim/ftplugin/maddy-conf.vim new file mode 100644 index 0000000..d5c49c6 --- /dev/null +++ b/dist/vim/ftplugin/maddy-conf.vim @@ -0,0 +1,8 @@ +setlocal commentstring=#\ %s + +" That is convention for maddy configs. Period. +" - fox.cpp (maddy developer) +setlocal expandtab +setlocal tabstop=4 +setlocal softtabstop=4 +setlocal shiftwidth=4 diff --git a/dist/vim/syntax/maddy-conf.vim b/dist/vim/syntax/maddy-conf.vim new file mode 100644 index 0000000..d59e799 --- /dev/null +++ b/dist/vim/syntax/maddy-conf.vim @@ -0,0 +1,225 @@ +" vim: noexpandtab ts=4 sw=4 + +if exists("b:current_syntax") + finish +endif + +" Lexer-defined rules +syn match maddyComment "#.*" +syn region maddyString start=+"+ skip=+\\\\\|\\"+ end=+"+ oneline + +syn region maddyBlock start="{" end="}" transparent fold + +hi def link maddyComment Comment +hi def link maddyString String + +" Parser-defined rules +syn match maddyMacroName "[a-z0-9_]" contained containedin=maddyMacro +syn match maddyMacro "$(.\{-})" contains=maddyMacroName + +syn match maddyMacroDefSign "=" contained +syn match maddyMacroDef "\^$([a-z0-9_]\{-})\s=\s.\+" contains=maddyMacro,maddyMacroDefSign + +hi def link maddyMacroName Identifier +hi def link maddyMacro Special +hi def link maddyMacroDefSign Special + +" config.Map values +syn keyword maddyBool yes no + +syn match maddyInt '\<\d\+\>' +syn match maddyInt '\<[-+]\d\+\>' +syn match maddyFloat '\<[-+]\d\+\.\d*\<' + +syn match maddyReference /[ \t]&[^ \t]\+/ms=s+1 contains=maddyReferenceSign +syn match maddyReferenceSign /&/ contained + +hi def link maddyBool Boolean +hi def link maddyInt Number +hi def link maddyFloat Float + +hi def link maddyReferenceSign Special + +" Module values + +" grep --no-file -E 'Register.*\(".+", ' **.go | sed -E 's/.+Register.*\("([^"]+)", .+/\1/' | sort -u +syn keyword maddyModule + \ checks + \ command + \ dane + \ dkim + \ dnsbl + \ dnssec + \ dummy + \ extauth + \ external + \ file + \ identity + \ imap + \ imap_filters + \ imapsql + \ limits + \ lmtp + \ loader + \ local_policy + \ milter + \ modifiers + \ msgpipeline + \ mtasts + \ mx_auth + \ pam + \ pass_table + \ plain_separate + \ queue + \ regexp + \ remote + \ replace_rcpt + \ replace_sender + \ require_matching_rdns + \ require_mx_record + \ require_tls + \ rspamd + \ shadow + \ smtp + \ sql_query + \ sql_table + \ static + \ submission + +syn keyword maddyDispatchDir + \ check + \ modify + \ default_source + \ source + \ default_destination + \ destination + \ reject + \ deliver_to + \ reroute + \ dmarc + +" grep --no-file -E 'cfg..+\(".+", ' **.go | sed -E 's/.+cfg..+\("([^"]+)", .+/\1/' | sort -u +syn keyword maddyModDir + \ add + \ add_header_action + \ allow_multiple_from + \ api_path + \ appendlimit + \ attempt_starttls + \ auth + \ autogenerated_msg_domain + \ body_canon + \ bounce + \ broken_sig_action + \ buffer + \ cache + \ case_insensitive + \ certs + \ check_early + \ client_ipv4 + \ client_ipv6 + \ compression + \ conn_max_idle_count + \ conn_max_idle_time + \ conn_reuse_limit + \ debug + \ defer_sender_reject + \ del + \ domains + \ driver + \ dsn + \ ehlo + \ endpoint + \ enforce_early + \ enforce_testing + \ entry + \ error_resp_action + \ expand_replaceholders + \ fail_action + \ fail_open + \ file + \ flags + \ force_ipv4 + \ fs_dir + \ fsstore + \ full_match + \ hash + \ header_canon + \ helper + \ hostname + \ imap_filter + \ init + \ insecure_auth + \ io_debug + \ io_error_action + \ io_errors + \ junk_mailbox + \ key_column + \ key_path + \ keys + \ limits + \ list + \ local_ip + \ location + \ lookup + \ mailfrom + \ max_logged_rcpt_errors + \ max_message_size + \ max_parallelism + \ max_received + \ max_recipients + \ max_tries + \ min_mx_level + \ min_tls_level + \ mx_auth + \ neutral_action + \ newkey_algo + \ none_action + \ no_sig_action + \ oversign_fields + \ pass + \ perdomain + \ permerr_action + \ quarantine_threshold + \ read_timeout + \ reject_threshold + \ relaxed_requiretls + \ required_fields + \ require_sender_match + \ require_tls + \ requiretls_override + \ responses + \ rewrite_subj_action + \ run_on + \ score + \ selector + \ set + \ settings_id + \ sig_expiry + \ sign_fields + \ sign_subdomains + \ softfail_action + \ SOME_action + \ source + \ sqlite3_busy_timeout + \ sqlite3_cache_size + \ sqlite3_exclusive_lock + \ storage + \ table + \ table_name + \ tag + \ target + \ targets + \ temperr_action + \ tls + \ tls_client + \ use_helper + \ user + \ value_column + \ write_timeout + +hi def link maddyModDir Identifier +hi def link maddyModule Identifier +hi def link maddyDispatchDir Identifier + +let b:current_syntax = "maddy" diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 0000000..d2bc346 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,81 @@ +# Docker + +Official Docker image is available from Docker Hub. + +It expects configuration file to be available at /data/maddy.conf. + +If /data is a Docker volume, then default configuration will be placed there +automatically. If it is used, then MADDY_HOSTNAME, MADDY_DOMAIN environment +variables control the host name and primary domain for the server. TLS +certificate should be placed in /data/tls/fullchain.pem, private key in +/data/tls/privkey.pem + +DKIM keys are generated in /data/dkim_keys directory. + +## Image tags + +- `latest` - A latest stable release. May contain breaking changes. +- `X.Y` - A specific feature branch, it is recommended to use these tags to + receive bugfixes without the risk of feature-related regressions or breaking + changes. +- `X.Y.Z` - A specific stable release + +## Ports + +All standard ports, as described in maddy docs. + +- `25` - SMTP inbound port. +- `465`, `587` - SMTP Submission ports +- `993`, `143` - IMAP4 ports + +## Volumes + +`/data` - maddy state directory. Databases, queues, etc are stored here. You +might want to mount a named volume there. The main configuration file is stored +here too (`/data/maddy.conf`). + +## Management utility + +To run management commands, create a temporary container with the same +/data directory and put the command after the image name, like this: + +``` +docker run --rm -it -v maddydata:/data foxcpp/maddy:0.7 creds create foxcpp@maddy.test +docker run --rm -it -v maddydata:/data foxcpp/maddy:0.7 imap-acct create foxcpp@maddy.test +``` + +Use the same image version as the running server. Things may break badly +otherwise. + +Note that, if you modify messages using maddy subcommands while the server is running - +you must ensure that /tmp from the server is accessible for the management +command. One way to it is to run it using `docker exec` instead of `docker run`: +``` +docker exec -it container_name_here maddy creds create foxcpp@maddy.test +``` + +## Build Tags + +Some Maddy features (such as automatic certificate management via ACME with [a non-default libdns provider](../reference/tls-acme/#dns-providers)) require build tags to be passed to Maddy's `build.sh`, as this is run in the Dockerfile you must compile your own Docker image. Build tags can be set via the docker build argument `ADDITIONAL_BUILD_TAGS` e.g. `docker build --build-arg ADDITIONAL_BUILD_TAGS="libdns_acmedns libdns_route53" -t yourorgname/maddy:yourtagname .`. + + +## TL;DR + +``` +docker volume create maddydata +docker run \ + --name maddy \ + -e MADDY_HOSTNAME=mx.maddy.test \ + -e MADDY_DOMAIN=maddy.test \ + -v maddydata:/data \ + -p 25:25 \ + -p 143:143 \ + -p 465:465 \ + -p 587:587 \ + -p 993:993 \ + foxcpp/maddy:0.7 +``` + +It will fail on first startup. Copy TLS certificate to /data/tls/fullchain.pem +and key to /data/tls/privkey.pem. Run the server again. Finish DNS configuration +(DKIM keys, etc) as described in [tutorials/setting-up/](../tutorials/setting-up/). diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 0000000..b9d755c --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,119 @@ +# Frequently Asked Questions + +## I configured maddy as recommended and gmail still puts my messages in spam + +Unfortunately, GMail policies are opaque so we cannot tell why this happens. + +Verify that you have a rDNS record set for the IP used +by sender server. Also some IPs may just happen to +have bad reputation - check it with various DNSBLs. In this +case you do not have much of a choice but to replace it. + +Additionally, you may try marking multiple messages sent from +your domain as "not spam" in GMail UI. + +## Message sending fails with `dial tcp X.X.X.X:25: connect: connection timed out` in log + +Your provider is blocking outbound SMTP traffic on port 25. + +You either have to ask them to unblock it or forward +all outbound messages via a "smart-host". + +## What is resource usage of maddy? + +For a small personal server, you do not need much more than a +single 1 GiB of RAM and disk space. + +## How to setup a catchall address? + +https://github.com/foxcpp/maddy/issues/243#issuecomment-655694512 + +## maddy command prints a "permission denied" error + +Run maddy command under the same user as maddy itself. +E.g. +``` +sudo -u maddy maddy creds ... +``` + +## How maddy compares to MailCow or Mail-In-The-Box? + +MailCow and MIAB are bundles of well-known email-related software configured to +work together. maddy is a single piece of software implementing subset of what +MailCow and MIAB offer. + +maddy offers more uniform configuration system, more lightweight implementation +and has no dependency on Docker or similar technologies for deployment. + +maddy may have more bugs than 20 years old battle-tested software. + +It is easier to get help with MailCow/MITB since underlying implementations +are well-understood and have active community. + +maddy has no Web interface for administration, that is currently done via CLI +utility. + +## How maddy IMAP server compares to WildDuck? + +Both are "more secure by definition": root access is not required, +implementation is in memory-safe language, etc. + +Both support message compression. + +Both have first-class Unicode/internationalization support. + +WildDuck may offer easier scalability options. maddy does not require you to +setup MongoDB and Redis servers, though. In fact, maddy in its default +configuration has no dependencies besides libc. + +maddy has less builtin authentication providers. This means no +app-specific passwords and all that WildDuck lists under point 4 on their +features page. + +maddy currently has no admin Web interface, all necessary DB changes are +performed via CLI utility. + +## How maddy SMTP server compares to ZoneMTA? + +maddy SMTP server has a lot of similarities to ZoneMTA. +Both have powerful mechanisms for message routing (although designed +differently). + +maddy does not require MongoDB server for deployment. + +maddy has no web interface for queue inspection. However, it can +easily inspected by looking at files in /var/lib/maddy. + +ZoneMTA has a number of features that may make it easier to integrate +with HTTP-based services. maddy speaks standard email protocols (SMTP, +Submission). + +## Is there a webmail? + +No, at least currently. + +I suggest you to check out [alps](https://git.sr.ht/~migadu/alps) if you +are fine with alpha-quality but extremely easy to deploy webmail. + +## Is there a content filter (spam filter)? + +No. maddy moves email messages around, it does not classify +them as bad or good with the notable exception of sender policies. + +It is possible to integrate rspamd using 'rspamd' module. Just add +`rspamd` line to `checks` in `local_routing`, it should just work +in most cases. + +## Is it production-ready? + +maddy is considered "beta" quality. Several people use it for personal email. + +## Single process makes it unreliable. This is dumb! + +This is a compromise between ease of management and reliability. Several +measures are implemented in code base in attempt to reduce possible effect +of bugs in one component. + +Besides, you are not required to use a single process, it is easy to launch +maddy with a non-default configuration path and connect multiple instances +together using off-the-shelf protocols. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..901f723 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,22 @@ +> Composable all-in-one mail server. + +Maddy Mail Server implements all functionality required to run a e-mail +server. It can send messages via SMTP (works as MTA), accept messages via SMTP +(works as MX) and store messages while providing access to them via IMAP. +In addition to that it implements auxiliary protocols that are mandatory +to keep email reasonably secure (DKIM, SPF, DMARC, DANE, MTA-STS). + +It replaces Postfix, Dovecot, OpenDKIM, OpenSPF, OpenDMARC and more with one +daemon with uniform configuration and minimal maintenance cost. + +**Note:** IMAP storage is "beta". If you are looking for stable and +feature-packed implementation you may want to use Dovecot instead. maddy still +can handle message delivery business. + +[![builds.sr.ht status](https://builds.sr.ht/~emersion/maddy.svg)](https://builds.sr.ht/~emersion/maddy?) +[![License text](https://img.shields.io/github/license/foxcpp/maddy)](https://github.com/foxcpp/maddy/blob/master/LICENSE) +[![Issues tracker](https://img.shields.io/github/issues/foxcpp/maddy)](https://github.com/foxcpp/maddy) + +* [Setup tutorial](https://maddy.email/tutorials/setting-up/) +* [IRC channel](https://webchat.oftc.net/?channels=maddy&uio=MT11bmRlZmluZWQb1) +* [Mailing list](https://lists.sr.ht/~foxcpp/maddy) diff --git a/docs/internals/quirks.md b/docs/internals/quirks.md new file mode 100644 index 0000000..bc343cc --- /dev/null +++ b/docs/internals/quirks.md @@ -0,0 +1,23 @@ +# Implementation quirks + +This page documents unusual behavior of the maddy protocols implementations. +Some of these problems break standards, some don't but still can hurt +interoperability. + +## SMTP + +- `for` field is never included in the `Received` header field. + + This is allowed by [RFC 2821]. + +## IMAP + +### `sql` + +- `\Recent` flag is not reset in all cases. + + This _does not_ break [RFC 3501]. Clients relying on it will work (much) less + efficiently. + +[RFC 2821]: https://tools.ietf.org/html/rfc2821 +[RFC 3501]: https://tools.ietf.org/html/rfc3501 diff --git a/docs/internals/specifications.md b/docs/internals/specifications.md new file mode 100644 index 0000000..c042cab --- /dev/null +++ b/docs/internals/specifications.md @@ -0,0 +1,291 @@ +# Followed specifications + +This page lists Internet Standards and other specifications followed by +maddy along with any known deviations. + + +## Message format + +- [RFC 2822] - Internet Message Format +- [RFC 2045] - Multipurpose Internet Mail Extensions (MIME) (part 1) +- [RFC 2046] - Multipurpose Internet Mail Extensions (MIME) (part 2) +- [RFC 2047] - Multipurpose Internet Mail Extensions (MIME) (part 3) +- [RFC 2048] - Multipurpose Internet Mail Extensions (MIME) (part 4) +- [RFC 2049] - Multipurpose Internet Mail Extensions (MIME) (part 5) +- [RFC 6532] - Internationalized Email Headers + +- [RFC 2183] - Communicating Presentation Information in Internet Messages: The + Content-Disposition Header Field + +## IMAP + +- [RFC 3501] - Internet Message Access Protocol - Version 4rev1 + * **Partial**: `\Recent` flag is not reset sometimes. +- [RFC 2152] - UTF-7 + +### Extensions + +- [RFC 2595] - Using TLS with IMAP, POP3 and ACAP +- [RFC 7889] - The IMAP APPENDLIMIT Extension +- [RFC 3348] - The Internet Message Action Protocol (IMAP4). Child Mailbox + Extension +- [RFC 6851] - Internet Message Access Protocol (IMAP) - MOVE Extension +- [RFC 6154] - IMAP LIST Extension for Special-Use Mailboxes + * **Partial**: Only SPECIAL-USE capability. +- [RFC 5255] - Internet Message Access Protocol Internationalization + * **Partial**: Only I18NLEVEL=1 capability. +- [RFC 4978] - The IMAP COMPRESS Extension +- [RFC 3691] - Internet Message Access Protocol (IMAP) UNSELECT command +- [RFC 2177] - IMAP4 IDLE command +- [RFC 7888] - IMAP4 Non-Synchronizing Literals + * LITERAL+ capability. +- [RFC 4959] - IMAP Extension for Simple Authentication and Security Layer + (SASL) Initial Client Response + +## SMTP + +- [RFC 2033] - Local Mail Transfer Protocol +- [RFC 5321] - Simple Mail Transfer Protocol +- [RFC 6409] - Message Submission for Mail + +### Extensions + +- [RFC 1870] - SMTP Service Extension for Message Size Declaration +- [RFC 2920] - SMTP Service Extension for Command Pipelining + * Server support only, not used by SMTP client +- [RFC 2034] - SMTP Service Extension for Returning Enhanced Error Codes +- [RFC 3207] - SMTP Service Extension for Secure SMTP over Transport Layer + Security +- [RFC 4954] - SMTP Service Extension for Authentication +- [RFC 6152] - SMTP Extension for 8-bit MIME +- [RFC 6531] - SMTP Extension for Internationalized Email + +### Misc + +- [RFC 6522] - The Multipart/Report Content Type for the Reporting of Mail + System Administrative Messages +- [RFC 3464] - An Extensible Message Format for Delivery Status Notifications +- [RFC 6533] - Internationalized Delivery Status and Disposition Notifications + +## Email security + +### User authentication + +- [RFC 4422] - Simple Authentication and Security Layer (SASL) +- [RFC 4616] - The PLAIN Simple Authentication and Security Layer (SASL) + Mechanism + +### Sender authentication + +- [RFC 6376] - DomainKeys Identified Mail (DKIM) Signatures +- [RFC 7001] - Message Header Field for Indicating Message Authentication Status +- [RFC 7208] - Sender Policy Framework (SPF) for Authorizing Use of Domains in + Email, Version 1 +- [RFC 7372] - Email Authentication Status Codes +- [RFC 7479] - Domain-based Message Authentication, Reporting, and Conformance + (DMARC) + * **Partial**: No report generation. +- [RFC 8301] - Cryptographic Algorithm and Key Usage Update to DomainKeys + Identified Mail (DKIM) +- [RFC 8463] - A New Cryptographic Signature Method for DomainKeys Identified + Mail (DKIM) +- [RFC 8616] - Email Authentication for Internationalized Mail + +### Recipient authentication + +- [RFC 4033] - DNS Security Introduction and Requirements +- [RFC 6698] - The DNS-Based Authentication of Named Entities (DANE) Transport + Layer Security (TLS) Protocol: TLSA +- [RFC 7672] - SMTP Security via Opportunistic DNS-Based Authentication of + Named Entities (DANE) Transport Layer Security (TLS) +- [RFC 8461] - SMTP MTA Strict Transport Security (MTA-STS) + +## Unicode, encodings, internationalization + +- [RFC 3492] - Punycode: A Bootstring encoding of Unicode for Internationalized + Domain Names in Applications (IDNA) +- [RFC 3629] - UTF-8, a transformation format of ISO 10646 +- [RFC 5890] - Internationalized Domain Names for Applications (IDNA): + Definitions and Document Framework +- [RFC 5891] - Internationalized Domain Names for Applications (IDNA): Protocol +- [RFC 7616] - Preparation, Enforcement, and Comparison of Internationalized + Strings Representing Usernames and Passwords +- [RFC 8264] - PRECIS Framework: Preparation, Enforcement, and Comparison of + Internationalized Strings in Application Protocols +- [Unicode 11.0.0] + - [UAX #15] - Unicode Normalization Forms + +There is a huge list of non-Unicode encodings supported by message parser used +for IMAP static cache and search. See [Unicode support](unicode.md) page for +details. + +## Misc + +- [RFC 5782] - DNS Blacklists and Whitelists + + +[GH 188]: https://github.com/foxcpp/maddy/issues/188 + +[RFC 2822]: https://tools.ietf.org/html/rfc2822 +[RFC 2045]: https://tools.ietf.org/html/rfc2045 +[RFC 2046]: https://tools.ietf.org/html/rfc2046 +[RFC 2047]: https://tools.ietf.org/html/rfc2047 +[RFC 2048]: https://tools.ietf.org/html/rfc2048 +[RFC 2049]: https://tools.ietf.org/html/rfc2049 +[RFC 6532]: https://tools.ietf.org/html/rfc6532 +[RFC 2183]: https://tools.ietf.org/html/rfc2183 +[RFC 3501]: https://tools.ietf.org/html/rfc3501 +[RFC 2152]: https://tools.ietf.org/html/rfc2152 +[RFC 2595]: https://tools.ietf.org/html/rfc2595 +[RFC 7889]: https://tools.ietf.org/html/rfc7889 +[RFC 3348]: https://tools.ietf.org/html/rfc3348 +[RFC 6851]: https://tools.ietf.org/html/rfc6851 +[RFC 6154]: https://tools.ietf.org/html/rfc6154 +[RFC 5255]: https://tools.ietf.org/html/rfc5255 +[RFC 4978]: https://tools.ietf.org/html/rfc4978 +[RFC 3691]: https://tools.ietf.org/html/rfc3691 +[RFC 2177]: https://tools.ietf.org/html/rfc2177 +[RFC 7888]: https://tools.ietf.org/html/rfc7888 +[RFC 4959]: https://tools.ietf.org/html/rfc4959 +[RFC 2033]: https://tools.ietf.org/html/rfc2033 +[RFC 5321]: https://tools.ietf.org/html/rfc5321 +[RFC 6409]: https://tools.ietf.org/html/rfc6409 +[RFC 1870]: https://tools.ietf.org/html/rfc1870 +[RFC 2920]: https://tools.ietf.org/html/rfc2920 +[RFC 2034]: https://tools.ietf.org/html/rfc2034 +[RFC 3207]: https://tools.ietf.org/html/rfc3207 +[RFC 4954]: https://tools.ietf.org/html/rfc4954 +[RFC 6152]: https://tools.ietf.org/html/rfc6152 +[RFC 6531]: https://tools.ietf.org/html/rfc6531 +[RFC 6522]: https://tools.ietf.org/html/rfc6522 +[RFC 3464]: https://tools.ietf.org/html/rfc3464 +[RFC 6533]: https://tools.ietf.org/html/rfc6533 +[RFC 4422]: https://tools.ietf.org/html/rfc4422 +[RFC 4616]: https://tools.ietf.org/html/rfc4616 +[RFC 6376]: https://tools.ietf.org/html/rfc6376 +[RFC 7001]: https://tools.ietf.org/html/rfc7001 +[RFC 7208]: https://tools.ietf.org/html/rfc7208 +[RFC 7372]: https://tools.ietf.org/html/rfc7372 +[RFC 7479]: https://tools.ietf.org/html/rfc7479 +[RFC 8301]: https://tools.ietf.org/html/rfc8301 +[RFC 8463]: https://tools.ietf.org/html/rfc8463 +[RFC 8616]: https://tools.ietf.org/html/rfc8616 +[RFC 4033]: https://tools.ietf.org/html/rfc4033 +[RFC 6698]: https://tools.ietf.org/html/rfc6698 +[RFC 7672]: https://tools.ietf.org/html/rfc7672 +[RFC 8461]: https://tools.ietf.org/html/rfc8461 +[RFC 3492]: https://tools.ietf.org/html/rfc3492 +[RFC 3629]: https://tools.ietf.org/html/rfc3629 +[RFC 5890]: https://tools.ietf.org/html/rfc5890 +[RFC 5891]: https://tools.ietf.org/html/rfc5891 +[RFC 7616]: https://tools.ietf.org/html/rfc7616 +[RFC 8264]: https://tools.ietf.org/html/rfc8264 +[RFC 5782]: https://tools.ietf.org/html/rfc5782 +[RFC 2822]: https://tools.ietf.org/html/rfc2822 +[RFC 2045]: https://tools.ietf.org/html/rfc2045 +[RFC 2046]: https://tools.ietf.org/html/rfc2046 +[RFC 2047]: https://tools.ietf.org/html/rfc2047 +[RFC 2048]: https://tools.ietf.org/html/rfc2048 +[RFC 2049]: https://tools.ietf.org/html/rfc2049 +[RFC 6532]: https://tools.ietf.org/html/rfc6532 +[RFC 3501]: https://tools.ietf.org/html/rfc3501 +[RFC 2595]: https://tools.ietf.org/html/rfc2595 +[RFC 7889]: https://tools.ietf.org/html/rfc7889 +[RFC 3348]: https://tools.ietf.org/html/rfc3348 +[RFC 6851]: https://tools.ietf.org/html/rfc6851 +[RFC 6154]: https://tools.ietf.org/html/rfc6154 +[RFC 5255]: https://tools.ietf.org/html/rfc5255 +[RFC 4978]: https://tools.ietf.org/html/rfc4978 +[RFC 3691]: https://tools.ietf.org/html/rfc3691 +[RFC 2177]: https://tools.ietf.org/html/rfc2177 +[RFC 7888]: https://tools.ietf.org/html/rfc7888 +[RFC 4959]: https://tools.ietf.org/html/rfc4959 +[RFC 2033]: https://tools.ietf.org/html/rfc2033 +[RFC 5321]: https://tools.ietf.org/html/rfc5321 +[RFC 6409]: https://tools.ietf.org/html/rfc6409 +[RFC 1870]: https://tools.ietf.org/html/rfc1870 +[RFC 2920]: https://tools.ietf.org/html/rfc2920 +[RFC 2034]: https://tools.ietf.org/html/rfc2034 +[RFC 3207]: https://tools.ietf.org/html/rfc3207 +[RFC 4954]: https://tools.ietf.org/html/rfc4954 +[RFC 6152]: https://tools.ietf.org/html/rfc6152 +[RFC 6531]: https://tools.ietf.org/html/rfc6531 +[RFC 6522]: https://tools.ietf.org/html/rfc6522 +[RFC 3464]: https://tools.ietf.org/html/rfc3464 +[RFC 6533]: https://tools.ietf.org/html/rfc6533 +[RFC 4422]: https://tools.ietf.org/html/rfc4422 +[RFC 4616]: https://tools.ietf.org/html/rfc4616 +[RFC 6376]: https://tools.ietf.org/html/rfc6376 +[RFC 7001]: https://tools.ietf.org/html/rfc7001 +[RFC 7208]: https://tools.ietf.org/html/rfc7208 +[RFC 7372]: https://tools.ietf.org/html/rfc7372 +[RFC 7479]: https://tools.ietf.org/html/rfc7479 +[RFC 8301]: https://tools.ietf.org/html/rfc8301 +[RFC 8463]: https://tools.ietf.org/html/rfc8463 +[RFC 8616]: https://tools.ietf.org/html/rfc8616 +[RFC 4033]: https://tools.ietf.org/html/rfc4033 +[RFC 6698]: https://tools.ietf.org/html/rfc6698 +[RFC 7672]: https://tools.ietf.org/html/rfc7672 +[RFC 8461]: https://tools.ietf.org/html/rfc8461 +[RFC 3492]: https://tools.ietf.org/html/rfc3492 +[RFC 3629]: https://tools.ietf.org/html/rfc3629 +[RFC 5890]: https://tools.ietf.org/html/rfc5890 +[RFC 5891]: https://tools.ietf.org/html/rfc5891 +[RFC 7616]: https://tools.ietf.org/html/rfc7616 +[RFC 8264]: https://tools.ietf.org/html/rfc8264 +[RFC 5782]: https://tools.ietf.org/html/rfc5782 +[RFC 2822]: https://tools.ietf.org/html/rfc2822 +[RFC 2045]: https://tools.ietf.org/html/rfc2045 +[RFC 2046]: https://tools.ietf.org/html/rfc2046 +[RFC 2047]: https://tools.ietf.org/html/rfc2047 +[RFC 2048]: https://tools.ietf.org/html/rfc2048 +[RFC 2049]: https://tools.ietf.org/html/rfc2049 +[RFC 6532]: https://tools.ietf.org/html/rfc6532 +[RFC 3501]: https://tools.ietf.org/html/rfc3501 +[RFC 2595]: https://tools.ietf.org/html/rfc2595 +[RFC 7889]: https://tools.ietf.org/html/rfc7889 +[RFC 3348]: https://tools.ietf.org/html/rfc3348 +[RFC 6851]: https://tools.ietf.org/html/rfc6851 +[RFC 6154]: https://tools.ietf.org/html/rfc6154 +[RFC 5255]: https://tools.ietf.org/html/rfc5255 +[RFC 4978]: https://tools.ietf.org/html/rfc4978 +[RFC 3691]: https://tools.ietf.org/html/rfc3691 +[RFC 2177]: https://tools.ietf.org/html/rfc2177 +[RFC 7888]: https://tools.ietf.org/html/rfc7888 +[RFC 4959]: https://tools.ietf.org/html/rfc4959 +[RFC 2033]: https://tools.ietf.org/html/rfc2033 +[RFC 5321]: https://tools.ietf.org/html/rfc5321 +[RFC 6409]: https://tools.ietf.org/html/rfc6409 +[RFC 1870]: https://tools.ietf.org/html/rfc1870 +[RFC 2920]: https://tools.ietf.org/html/rfc2920 +[RFC 2034]: https://tools.ietf.org/html/rfc2034 +[RFC 3207]: https://tools.ietf.org/html/rfc3207 +[RFC 4954]: https://tools.ietf.org/html/rfc4954 +[RFC 6152]: https://tools.ietf.org/html/rfc6152 +[RFC 6531]: https://tools.ietf.org/html/rfc6531 +[RFC 6522]: https://tools.ietf.org/html/rfc6522 +[RFC 3464]: https://tools.ietf.org/html/rfc3464 +[RFC 6533]: https://tools.ietf.org/html/rfc6533 +[RFC 4422]: https://tools.ietf.org/html/rfc4422 +[RFC 4616]: https://tools.ietf.org/html/rfc4616 +[RFC 6376]: https://tools.ietf.org/html/rfc6376 +[RFC 8301]: https://tools.ietf.org/html/rfc8301 +[RFC 8463]: https://tools.ietf.org/html/rfc8463 +[RFC 7208]: https://tools.ietf.org/html/rfc7208 +[RFC 7372]: https://tools.ietf.org/html/rfc7372 +[RFC 7479]: https://tools.ietf.org/html/rfc7479 +[RFC 8616]: https://tools.ietf.org/html/rfc8616 +[RFC 4033]: https://tools.ietf.org/html/rfc4033 +[RFC 6698]: https://tools.ietf.org/html/rfc6698 +[RFC 7672]: https://tools.ietf.org/html/rfc7672 +[RFC 8461]: https://tools.ietf.org/html/rfc8461 +[RFC 3492]: https://tools.ietf.org/html/rfc3492 +[RFC 3629]: https://tools.ietf.org/html/rfc3629 +[RFC 5890]: https://tools.ietf.org/html/rfc5890 +[RFC 5891]: https://tools.ietf.org/html/rfc5891 +[RFC 7616]: https://tools.ietf.org/html/rfc7616 +[RFC 8264]: https://tools.ietf.org/html/rfc8264 +[RFC 5782]: https://tools.ietf.org/html/rfc5782 + +[Unicode 11.0.0]: https://www.unicode.org/versions/components-11.0.0.html +[UAX #15]: https://unicode.org/reports/tr15/ diff --git a/docs/internals/sqlite.md b/docs/internals/sqlite.md new file mode 100644 index 0000000..8a397fd --- /dev/null +++ b/docs/internals/sqlite.md @@ -0,0 +1,49 @@ +# maddy & SQLite + +SQLite is a perfect choice for small deployments because no additional +configuration is required to get started. It is recommended for cases when you +have less than 10 mail accounts. + +**Note: SQLite requires DB-wide locking for writing, it means that multiple +messages can't be accepted in parallel. This is not the case for server-based +RDBMS where maddy can accept multiple messages in parallel even for a single +mailbox.** + +## WAL mode + +maddy forces WAL journal mode for SQLite. This makes things reasonably fast and +reduces locking contention which may be important for a typical mail server. + +maddy uses increased WAL autocheckpoint interval. This means that while +maintaining a high write throughput, maddy will have to stop for a bit (0.5-1 +second) every time 78 MiB is written to the DB (with default configuration it +is 15 MiB). + +Note that when moving the database around you need to move WAL journal (`-wal`) +and shared memory (`-shm`) files as well, otherwise some changes to the DB will +be lost. + +## Query planner statistics + +maddy updates query planner statistics on shutdown and every 5 hours. It +provides query planner with information to access the database in more +efficient way because go-imap-sql schema does use a few so called "low-quality +indexes". + +## Auto-vacuum + +maddy turns on SQLite auto-vacuum feature. This means that database file size +will shrink when data is removed (compared to default when it remains unused). + +## Manual vacuuming + +Auto-vacuuming can lead to database fragmentation and thus reduce the read +performance. To do manual vacuum operation to repack and defragment database +file, install the SQLite3 console utility and run the following commands: +``` +sqlite3 -cmd 'vacuum' database_file_path_here.db +sqlite3 -cmd 'pragma wal_checkpoint(truncate)' database_file_path_here.db +``` + +It will take some time to complete, you can close the utility when the +`sqlite>` prompt appears. diff --git a/docs/internals/unicode.md b/docs/internals/unicode.md new file mode 100644 index 0000000..9199762 --- /dev/null +++ b/docs/internals/unicode.md @@ -0,0 +1,96 @@ +# Unicode support + +maddy has the first-class Unicode support in all components (modules). You do +not have to take any actions to make it work with internationalized domains, +mailbox names or non-ASCII message headers. + +Internally, all text fields in maddy are represented in UTF-8 and handled using +Unicode-aware operations for comparisons, case-folding and so on. + +## Non-ASCII data in message headers and bodies + +maddy SMTP implementation does not care about encodings used in MIME headers or +in `Content-Type text/*` charset field. + +However, local IMAP storage implementation needs to perform certain operations +on header contents. This is mostly about SEARCH functionality. For IMAP search +to work correctly, the message body and headers should use one of the following +encodings: + +- ASCII +- UTF-8 +- ISO-8859-1, 2, 3, 4, 9, 10, 13, 14, 15 or 16 +- Windows-1250, 1251 or 1252 (aka Code Page 1250 and so on) +- KOI8-R +- ~~HZGB2312~~, GB18030 +- GBK (aka Code Page 936) +- Shift JIS (aka Code Page 932 or Windows-31J) +- Big-5 (aka Code Page 950) +- EUC-JP +- ISO-2022-JP + +_Support for HZGB2312 is currently disabled due to bugs with security +implications._ + +If mailbox includes a message with any encoding not listed here, it will not +be returned in search results for any request. + +Behavior regarding handling of non-Unicode encodings is not considered stable +and may change between versions (including removal of supported encodings). If +you need your stuff to work correctly - start using UTF-8. + +## Configuration files + +maddy configuration files are assumed to be encoded in UTF-8. Use of any other +encoding will break stuff, do not do it. + +Domain names (e.g. in hostname directive or pipeline rules) can be represented +using the ACE form (aka Punycode). They will be converted to the Unicode form +internally. + +## Local credentials + +'sql' storage backend and authentication provider enforce a number of additional +constraints on used account names. + +PRECIS UsernameCaseMapped profile is enforced for local email addresses. +It limits the use of control and Bidi characters to make sure the used value +can be represented consistently in a variety of contexts. On top of that, the +address is case-folded and normalized to the NFC form for consistent internal +handling. + +PRECIS OpaqueString profile is enforced for passwords. Less strict rules are +applied here. Runs of Unicode whitespace characters are replaced with a single +ASCII space. NFC normalization is applied afterwards. If the resulting string +is empty - the password is not accepted. + +Both profiles are defined in RFC 8265, consult it for details. + +## Protocol support + +### SMTPUTF8 extension + +maddy SMTP implementation includes support for the SMTPUTF8 extension as +defined in RFC 6531. + +This means maddy can handle internationalized mailbox and domain names in MAIL +FROM, RCPT TO commands both for outbound and inbound delivery. + +maddy will not accept messages with non-ASCII envelope addresses unless +SMTPUTF8 support is requested. If a message with SMTPUTF8 flag set is forwarded +to a server without SMTPUTF8 support, delivery will fail unless it is possible +to represent envelope addresses in the ASCII form (only domains use Unicode and +they can be converted to Punycode). Contents of message body (and header) are +not considered and always accepted and sent as-is, no automatic downgrading or +reencoding is done. + +### IMAP UTF8, I18NLEVEL extensions + +Currently, maddy does not include support for UTF8 and I18NLEVEL IMAP +extensions. However, it is not a problem that can prevent it from correctly +handling UTF-8 messages (or even messages in other non-ASCII encodings +mentioned above). + +Clients that want to implement proper handling for Unicode strings may assume +maddy does not handle them properly in e.g. SEARCH commands and so such clients +may download messages and process them locally. diff --git a/docs/man/.gitignore b/docs/man/.gitignore new file mode 100644 index 0000000..ad8dbc8 --- /dev/null +++ b/docs/man/.gitignore @@ -0,0 +1 @@ +_generated_*.md diff --git a/docs/man/README.md b/docs/man/README.md new file mode 100644 index 0000000..2363c6e --- /dev/null +++ b/docs/man/README.md @@ -0,0 +1,16 @@ +maddy manual pages +------------------- + +The reference documentation is maintained in the scdoc format and is compiled +into a set of Unix man pages viewable using the standard `man` utility. + +See https://git.sr.ht/~sircmpwn/scdoc for information about the tool used to +build pages. +It can be used as follows: +``` +scdoc < maddy-filters.5.scd > maddy-filters.5 +man ./maddy-filters.5 +``` + +build.sh script in the repo root compiles and installs man pages if the scdoc +utility is installed in the system. diff --git a/docs/man/maddy.1.scd b/docs/man/maddy.1.scd new file mode 100644 index 0000000..9ae63d5 --- /dev/null +++ b/docs/man/maddy.1.scd @@ -0,0 +1,41 @@ +maddy(1) "maddy mail server" "maddy reference documentation" + +; TITLE Command line arguments + +# Name + +maddy - Composable all-in-one mail server. + +# Synopsis + +*maddy* [options...] + +# Description + +Maddy is Mail Transfer agent (MTA), Mail Delivery Agent (MDA), Mail Submission +Agent (MSA), IMAP server and a set of other essential protocols/schemes +necessary to run secure email server implemented in one executable. + +# Command line arguments + +*-h, -help* + Show help message and exit. + +*-config* _path_ + Path to the configuration file. Default is /etc/maddy/maddy.conf. + +*-libexec* _path_ + Path to the libexec directory. Helper executables will be searched here. + Default is /usr/lib/maddy. + +*-log* _targets..._ + Comma-separated list of logging targets. Valid values are the same as the + 'log' config directive. Affects logging before configuration parsing + completes and after it, if the different value is not specified in the + configuration. + +*-debug* + Enable debug log. You want to use it when reporting bugs. + +*-v* + Print version & build metadata. diff --git a/docs/man/prepare_md.py b/docs/man/prepare_md.py new file mode 100644 index 0000000..9aeb335 --- /dev/null +++ b/docs/man/prepare_md.py @@ -0,0 +1,57 @@ +#!/usr/bin/python3 + +""" +This script does all necessary pre-processing to convert scdoc format into +Markdown. + +Usage: + prepare_md.py < in > out + prepare_md.py file1 file2 file3 + Converts into _generated_file1.md, etc. +""" + +import sys +import re + +anchor_escape = str.maketrans(r' #()./\+-_', '__________') + +def prepare(r, w): + new_lines = list() + title = str() + previous_h1_anchor = '' + + inside_literal = False + + for line in r: + if not inside_literal: + if line.startswith('; TITLE ') and title == '': + title = line[8:] + if line[0] == ';': + continue + # turn *page*(1) into [**page(1)**](../_generated_page.1) + line = re.sub(r'\*(.+?)\*\(([0-9])\)', r'[*\1(\2)*](../_generated_\1.\2)', line) + # *aaa* => **aaa** + line = re.sub(r'\*(.+?)\*', r'**\1**', line) + # remove ++ from line endings + line = re.sub(r'\+\+$', '
', line) + # turn whatever looks like a link into one + line = re.sub(r'(https://[^ \)\(\\]+[a-z0-9_\-])', r'[\1](\1)', line) + # escape underscores inside words + line = re.sub(r'([^ ])_([^ ])', r'\1\\_\2', line) + + if line.startswith('```'): + inside_literal = not inside_literal + + new_lines.append(line) + + if title != '': + print('#', title, file=w) + + print(''.join(new_lines[1:]), file=w) + +if len(sys.argv) == 1: + prepare(sys.stdin, sys.stdout) +else: + for f in sys.argv[1:]: + new_name = '_generated_' + f[:-4] + '.md' + prepare(open(f, 'r'), open(new_name, 'w')) diff --git a/docs/multiple-domains.md b/docs/multiple-domains.md new file mode 100644 index 0000000..46fabf0 --- /dev/null +++ b/docs/multiple-domains.md @@ -0,0 +1,157 @@ +# Multiple domains configuration + +By default, maddy uses email addresses as account identifiers for both +authentication and storage purposes. Therefore, account named `user@example.org` +is completely independent from `user@example.com`. They must be created +separately, may have different credentials and have separate IMAP mailboxes. + +This makes it extremely easy to setup maddy to manage multiple otherwise +independent domains. + +Default configuration file contains two macros - `$(primary_domain)` and +`$(local_domains)`. They are used to used in several places thorough the +file to configure message routing, security checks, etc. + +In general, you should just add all domains you want maddy to manage to +`$(local_domains)`, like this: +``` +$(primary_domain) = example.org +$(local_domains) = $(primary_domain) example.com +``` +Note that you need to pick one domain as a "primary" for use in +auto-generated messages. + +With that done, you can create accounts using both domains in the name, send +and receive messages and so on. Do not forget to configure corresponding SPF, +DMARC and MTA-STS records as was recommended in +the [introduction tutorial](tutorials/setting-up.md). + +Also note that you do not really need a separate TLS certificate for each +managed domain. You can have one hostname e.g. mail.example.org set as an MX +record for multiple domains. + +**If you want multiple domains to share username namespace**, you should change +several more options. + +You can make "user@example.org" and "user@example.com" users share the same +credentials of user "user" but have different IMAP mailboxes ("user@example.org" +and "user@example.com" correspondingly). For that, it is enough to set `auth_map` +globally to use `email_localpart` table: +``` +auth_map email_localpart +``` +This way, when user logs in as "user@example.org", "user" will be passed +to the authentication provider, but "user@example.org" will be passed to the +storage backend. You should create accounts like this: +``` +maddy creds create user +maddy imap-acct create user@example.org +maddy imap-acct create user@example.com +``` + +**If you want accounts to also share the same IMAP storage of account named +"user"**, you can set `storage_map` in IMAP endpoint and `delivery_map` in +storage backend to use `email_locapart`: +``` +storage.imapsql local_mailboxes { + ... + delivery_map email_localpart # deliver "user@*" to "user" +} +imap tls://0.0.0.0:993 { + ... + storage &local_mailboxes + ... + storage_map email_localpart # "user@*" accesses "user" mailbox +} +``` + +You also might want to make it possible to log in without +specifying a domain at all. In this case, use `email_localpart_optional` for +both `auth_map` and `storage_map`. + +You also need to make `authorize_sender` check (used in `submission` endpoint) +accept non-email usernames: +``` +authorize_sender { + ... + user_to_email chain { + step email_localpart_optional # remove domain from username if present + step email_with_domain $(local_domains) # expand username with all allowed domains + } +} +``` + +## TL;DR + +Your options: + +**"user@example.org" and "user@example.com" have distinct credentials and +distinct mailboxes.** + +``` +$(primary_domain) = example.org +$(local_domains) = example.org example.com +``` + +Create accounts as: + +```shell +maddy creds create user@example.org +maddy imap-acct create user@example.org +maddy creds create user@example.com +maddy imap-acct create user@example.com +``` + +**"user@example.org" and "user@example.com" have same credentials but +distinct mailboxes.** + +``` +$(primary_domain) = example.org +$(local_domains) = example.org example.com +auth_map email_localpart +``` + +Create accounts as: +```shell +maddy creds create user +maddy imap-acct create user@example.org +maddy imap-acct create user@example.com +``` + +**"user@example.org", "user@example.com", "user" have same credentials and same +mailboxes.** + +``` + $(primary_domain) = example.org + $(local_domains) = example.org example.com + auth_map email_localpart_optional # authenticating as "user@*" checks credentials for "user" + + storage.imapsql local_mailboxes { + ... + delivery_map email_localpart_optional # deliver "user@*" to "user" mailbox + } + + imap tls://0.0.0.0:993 { + ... + storage_map email_localpart_optional # authenticating as "user@*" accesses "user" mailboxes + } + + submission tls://0.0.0.0:465 { + check { + authorize_sender { + ... + user_to_email chain { + step email_localpart_optional # remove domain from username if present + step email_with_domain $(local_domains) # expand username with all allowed domains + } + } + } + ... + } +``` + +Create accounts as: +```shell +maddy creds create user +maddy imap-acct create user +``` diff --git a/docs/reference/auth/dovecot_sasl.md b/docs/reference/auth/dovecot_sasl.md new file mode 100644 index 0000000..919d42b --- /dev/null +++ b/docs/reference/auth/dovecot_sasl.md @@ -0,0 +1,26 @@ +# Dovecot SASL + +The 'auth.dovecot_sasl' module implements the client side of the Dovecot +authentication protocol, allowing maddy to use it as a credentials source. + +Currently SASL mechanisms support is limited to mechanisms supported by maddy +so you cannot get e.g. SCRAM-MD5 this way. + +``` +auth.dovecot_sasl { + endpoint unix://socket_path +} + +dovecot_sasl unix://socket_path +``` + +## Configuration directives + +### endpoint _schema://address_ +Default: not set + +Set the address to use to contact Dovecot SASL server in the standard endpoint +format. + +`tcp://10.0.0.1:2222` for TCP, `unix:///var/lib/dovecot/auth.sock` for Unix +domain sockets. diff --git a/docs/reference/auth/external.md b/docs/reference/auth/external.md new file mode 100644 index 0000000..9b9659e --- /dev/null +++ b/docs/reference/auth/external.md @@ -0,0 +1,52 @@ +# System command + +auth.external module for authentication using external helper binary. It looks for binary +named `maddy-auth-helper` in $PATH and libexecdir and uses it for authentication +using username/password pair. + +The protocol is very simple: +Program is launched for each authentication. Username and password are written +to stdin, adding \n to the end. If binary exits with 0 status code - +authentication is considered successful. If the status code is 1 - +authentication is failed. If the status code is 2 - another unrelated error has +happened. Additional information should be written to stderr. + +``` +auth.external { + helper /usr/bin/ldap-helper + perdomain no + domains example.org +} +``` + +## Configuration directives + +### helper _file_path_ + +**Required.**
+Location of the helper binary. + +--- + +### perdomain _boolean_ +Default: `no` + +Don't remove domain part of username when authenticating and require it to be +present. Can be used if you want user@domain1 and user@domain2 to be different +accounts. + +--- + +### domains _domains..._ +Default: not specified + +Domains that should be allowed in username during authentication. + +For example, if 'domains' is set to "domain1 domain2", then +username, username@domain1 and username@domain2 will be accepted as valid login +name in addition to just username. + +If used without 'perdomain', domain part will be removed from login before +check with underlying auth. mechanism. If 'perdomain' is set, then +domains must be also set and domain part **will not** be removed before check. + diff --git a/docs/reference/auth/ldap.md b/docs/reference/auth/ldap.md new file mode 100644 index 0000000..a4ced55 --- /dev/null +++ b/docs/reference/auth/ldap.md @@ -0,0 +1,130 @@ +# LDAP BindDN + +maddy supports authentication via LDAP using DN binding. Passwords are verified +by the LDAP server. + +maddy needs to know the DN to use for binding. It can be obtained either by +directory search or template . + +Note that storage backends conventionally use email addresses, if you use +non-email identifiers as usernames then you should map them onto +emails on delivery by using `auth_map` (see documentation page for used storage backend). + +auth.ldap also can be a used as a table module. This way you can check +whether the account exists. It works only if DN template is not used. + +``` +auth.ldap { + urls ldap://maddy.test:389 + + # Specify initial bind credentials. Not required ('bind off') + # if DN template is used. + bind plain "cn=maddy,ou=people,dc=maddy,dc=test" "123456" + + # Specify DN template to skip lookup. + dn_template "cn={username},ou=people,dc=maddy,dc=test" + + # Specify base_dn and filter to lookup DN. + base_dn "ou=people,dc=maddy,dc=test" + filter "(&(objectClass=posixAccount)(uid={username}))" + + tls_client { ... } + starttls off + debug off + connect_timeout 1m +} +``` +``` +auth.ldap ldap://maddy.test.389 { + ... +} +``` + +## Configuration directives + +### urls _servers..._ + +**Required.** + +URLs of the directory servers to use. First available server +is used - no load-balancing is done. + +URLs should use `ldap://`, `ldaps://`, `ldapi://` schemes. + +--- + +### bind `off` | `unauth` | `external` | `plain` _username_ _password_ + +Default: `off` + +Credentials to use for initial binding. Required if DN lookup is used. + +`unauth` performs unauthenticated bind. `external` performs external binding +which is useful for Unix socket connections (`ldapi://`) or TLS client certificate +authentication (cert. is set using tls_client directive). `plain` performs a +simple bind using provided credentials. + +--- + +### dn_template _template_ + +DN template to use for binding. `{username}` is replaced with the +username specified by the user. + +--- + +### base_dn _dn_ + +Base DN to use for lookup. + +--- + +### filter _str_ + +DN lookup filter. `{username}` is replaced with the username specified +by the user. + +Example: + +``` +(&(objectClass=posixAccount)(uid={username})) +``` + +Example (using ActiveDirectory): + +``` +(&(objectCategory=Person)(memberOf=CN=user-group,OU=example,DC=example,DC=org)(sAMAccountName={username})(!(UserAccountControl:1.2.840.113556.1.4.803:=2))) +``` + +Example: + +``` +(&(objectClass=Person)(mail={username})) +``` + +--- + +### starttls _bool_ +Default: `off` + +Whether to upgrade connection to TLS using STARTTLS. + +--- + +### tls_client { ... } + +Advanced TLS client configuration. See [TLS configuration / Client](/reference/tls/#client) for details. + +--- + +### connect_timeout _duration_ +Default: `1m` + +Timeout for initial connection to the directory server. + +--- + +### request_timeout _duration_ +Default: `1m` + +Timeout for each request (binding, lookup). diff --git a/docs/reference/auth/netauth.md b/docs/reference/auth/netauth.md new file mode 100644 index 0000000..2664d41 --- /dev/null +++ b/docs/reference/auth/netauth.md @@ -0,0 +1,48 @@ +# Native NetAuth + +maddy supports authentication via NetAuth using direct entity +authentication checks. Passwords are verified by the NetAuth server. + +maddy needs to know the Entity ID to use for authentication. It must +match the string the user provides for the Local Atom part of their +mail address. + +Note that storage backends conventionally use email addresses. Since +NetAuth recommends *nix compatible usernames, you will need to map the +email identifiers to NetAuth Entity IDs using `auth_map` (see +documentation page for used storage backend). + +auth.netauth also can be used as a table module. This way you can +check whether the account exists. + +Note that the configuration fragment provided below is very sparse. +This is because NetAuth expects to read most of its common +configuration values from the system NetAuth config file located at +`/etc/netauth/config.toml`. + +``` +auth.netauth { + require_group "maddy-users" + debug off +} +``` + +``` +auth.netauth {} +``` + +## Configuration directives + +### require_group _group_ + +Optional. + +Group that entities must possess to be able to use maddy services. +This can be used to provide email to just a subset of the entities +present in NetAuth. + +--- + +### debug `on` | `off` + +Default: `off` diff --git a/docs/reference/auth/pam.md b/docs/reference/auth/pam.md new file mode 100644 index 0000000..89f0f3e --- /dev/null +++ b/docs/reference/auth/pam.md @@ -0,0 +1,48 @@ +# PAM + +auth.pam module implements authentication using libpam. Alternatively it can be configured to +use helper binary like auth.external module does. + +maddy should be built with libpam build tag to use this module without +'use_helper' directive. + +``` +go get -tags 'libpam' ... +``` + +``` +auth.pam { + debug no + use_helper no +} +``` + +## Configuration directives + +### debug _boolean_ +Default: `no` + +Enable verbose logging for all modules. You don't need that unless you are +reporting a bug. + +--- + +### use_helper _boolean_ +Default: `no` + +Use `LibexecDirectory/maddy-pam-helper` instead of directly calling libpam. +You need to use that if: + +1. maddy is not compiled with libpam, but `maddy-pam-helper` is built separately. +2. maddy is running as an unprivileged user and used PAM configuration requires additional privileges (e.g. when using system accounts). + +For 2, you need to make `maddy-pam-helper` binary setuid, see +README.md in source tree for details. + +TL;DR (assuming you have the maddy group): + +``` +chown root:maddy /usr/lib/maddy/maddy-pam-helper +chmod u+xs,g+x,o-x /usr/lib/maddy/maddy-pam-helper +``` + diff --git a/docs/reference/auth/pass_table.md b/docs/reference/auth/pass_table.md new file mode 100644 index 0000000..39fea6f --- /dev/null +++ b/docs/reference/auth/pass_table.md @@ -0,0 +1,44 @@ +# Password table + +auth.pass_table module implements username:password authentication by looking up the +password hash using a table module (maddy-tables(5)). It can be used +to load user credentials from text file (via table.file module) or SQL query +(via table.sql_table module). + + +Definition: +``` +auth.pass_table [block name] { + table + +} +``` +Shortened variant for inline use: +``` +pass_table
[table arguments] { + [additional table config] +} +``` + +Example, read username:password pair from the text file: +``` +smtp tcp://0.0.0.0:587 { + auth pass_table file /etc/maddy/smtp_passwd + ... +} +``` + +## Password hashes + +pass_table expects the used table to contain certain structured values with +hash algorithm name, salt and other necessary parameters. + +You should use `maddy hash` command to generate suitable values. +See `maddy hash --help` for details. + +## maddy creds + +If the underlying table is a "mutable" table (see maddy-tables(5)) then +the `maddy creds` command can be used to modify the underlying tables +via pass_table module. It will act on a "local credentials store" and will write +appropriate hash values to the table. diff --git a/docs/reference/auth/plain_separate.md b/docs/reference/auth/plain_separate.md new file mode 100644 index 0000000..f5b5766 --- /dev/null +++ b/docs/reference/auth/plain_separate.md @@ -0,0 +1,45 @@ +# Separate username and password lookup + +auth.plain_separate module implements authentication using username:password pairs but can +use zero or more "table modules" (maddy-tables(5)) and one or more +authentication providers to verify credentials. + +``` +auth.plain_separate { + user ... + user ... + ... + pass ... + pass ... + ... +} +``` + +How it works: +- Initial username input is normalized using PRECIS UsernameCaseMapped profile. +- Each table specified with the 'user' directive looked up using normalized + username. If match is not found in any table, authentication fails. +- Each authentication provider specified with the 'pass' directive is tried. + If authentication with all providers fails - an error is returned. + +## Configuration directives + +### user _table-module_ + +Configuration block for any module from maddy-tables(5) can be used here. + +Example: + +``` +user file /etc/maddy/allowed_users +``` + +--- + +### pass _auth-provider_ + +Configuration block for any auth. provider module can be used here, even +'plain_split' itself. + +The used auth. provider must provide username:password pair-based +authentication. diff --git a/docs/reference/auth/shadow.md b/docs/reference/auth/shadow.md new file mode 100644 index 0000000..0fc3e89 --- /dev/null +++ b/docs/reference/auth/shadow.md @@ -0,0 +1,40 @@ +# /etc/shadow + +auth.shadow module implements authentication by reading /etc/shadow. Alternatively it can be +configured to use helper binary like auth.external does. + +``` +auth.shadow { + debug no + use_helper no +} +``` + +## Configuration directives + +### debug _boolean_ + +Default: `no` + +Enable verbose logging for all modules. You don't need that unless you are +reporting a bug. + +--- + +### use_helper _boolean_ +Default: `no` + +Use `LibexecDirectory/maddy-shadow-helper` instead of directly reading `/etc/shadow`. +You need to use that if maddy is running as an unprivileged user +privileges (e.g. when using system accounts). + +You need to make `maddy-shadow-helper` binary setuid, see +cmd/maddy-shadow-helper/README.md in source tree for details. + +TL;DR (assuming you have maddy group): + +``` +chown root:maddy /usr/lib/maddy/maddy-shadow-helper +chmod u+xs,g+x,o-x /usr/lib/maddy/maddy-shadow-helper +``` + diff --git a/docs/reference/blob/fs.md b/docs/reference/blob/fs.md new file mode 100644 index 0000000..ef94b54 --- /dev/null +++ b/docs/reference/blob/fs.md @@ -0,0 +1,23 @@ +# Filesystem + +This module stores message bodies in a file system directory. + +``` +storage.blob.fs { + root +} +``` + +``` +storage.blob.fs +``` + +## Configuration directives + +### root _path_ +Default: not set + +Path to the FS directory. Must be readable and writable by the server process. +If it does not exist - it will be created (parent directory should be writable +for this). Relative paths are interpreted relatively to server state directory. + diff --git a/docs/reference/blob/s3.md b/docs/reference/blob/s3.md new file mode 100644 index 0000000..54b6a4e --- /dev/null +++ b/docs/reference/blob/s3.md @@ -0,0 +1,98 @@ +# Amazon S3 + +storage.blob.s3 module stores messages bodies in a bucket on S3-compatible storage. + +``` +storage.blob.s3 { + endpoint play.min.io + secure yes + access_key "Q3AM3UQ867SPQQA43P2F" + secret_key "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" + bucket maddy-test + + # optional + region eu-central-1 + object_prefix maddy/ + creds access_key +} +``` + +Example: + +``` +storage.imapsql local_mailboxes { + ... + msg_store s3 { + endpoint s3.amazonaws.com + access_key "..." + secret_key "..." + bucket maddy-messages + region us-west-2 + creds access_key + } +} +``` + +## Configuration directives + +### endpoint _address:port_ + +**Required**. + +Root S3 endpoint. e.g. `s3.amazonaws.com` + +--- + +### secure _boolean_ +Default: `yes` + +Whether TLS should be used. + +--- + +### access_key _string_
secret_key _string_ + +**Required**. + +Static S3 credentials. + +--- + +### bucket _name_ + +**Required**. + +S3 bucket name. The bucket must exist and +be read-writable. + +--- + +### region _string_ +Default: not set + +S3 bucket location. May be called "endpoint" in some manuals. + +--- + +### object_prefix _string_ +Default: empty string + +String to add to all keys stored by maddy. + +Can be useful when S3 is used as a file system. + +--- + +### creds `access_key` | `file_minio` | `file_aws` | `iam` +Default: `access_key` + +Credentials to use for accessing the S3 Bucket. + +Credential Types: + + - `access_key`: use AWS access key and secret access key + - `file_minio`: use credentials for Minio present at ~/.mc/config.json + - `file_aws`: use credentials for AWS S3 present at ~/.aws/credentials + - `iam`: use AWS IAM instance profile for credentials. + +By default, access_key is used with the access key and secret access key present in the config. diff --git a/docs/reference/checks/actions.md b/docs/reference/checks/actions.md new file mode 100644 index 0000000..d9e9f9c --- /dev/null +++ b/docs/reference/checks/actions.md @@ -0,0 +1,21 @@ +# Check actions + +When a certain check module thinks the message is "bad", it takes some actions +depending on its configuration. Most checks follow the same configuration +structure and allow following actions to be taken on check failure: + +- Do nothing (`action ignore`) + +Useful for testing deployment of new checks. Check failures are still logged +but they have no effect on message delivery. + +- Reject the message (`action reject`) + +Reject the message at connection time. No bounce is generated locally. + +- Quarantine the message (`action quarantine`) + +Mark message as 'quarantined'. If message is then delivered to the local +storage, the storage backend can place the message in the 'Junk' mailbox. +Another thing to keep in mind that 'target.remote' module +will refuse to send quarantined messages. \ No newline at end of file diff --git a/docs/reference/checks/authorize_sender.md b/docs/reference/checks/authorize_sender.md new file mode 100644 index 0000000..4ddd786 --- /dev/null +++ b/docs/reference/checks/authorize_sender.md @@ -0,0 +1,132 @@ +# MAIL FROM and From authorization + +Module check.authorize_sender verifies that envelope and header sender addresses belong +to the authenticated user. Address ownership is established via table +that maps each user account to a email address it is allowed to use. +There are some special cases, see `user_to_email` description below. + +``` +check.authorize_sender { + prepare_email identity + user_to_email identity + check_header yes + + unauth_action reject + no_match_action reject + malformed_action reject + err_action reject + + auth_normalize auto + from_normalize auto +} +``` +``` +check { + authorize_sender { ... } +} +``` + +## Configuration directives + +### user_to_email _table_ +Default: `identity` + +Table that maps authorization username to the list of sender emails +the user is allowed to use. + +In additional to email addresses, the table can contain domain names or +special string "\*" as a value. If the value is a domain - user +will be allowed to use any mailbox within it as a sender address. +If it is "\*" - user will be allowed to use any address. + +By default, table.identity is used, meaning that username should +be equal to the sender email. + +Before username is looked up via the table, normalization algorithm +defined by auth_normalize is applied to it. + +--- + +### prepare_email _table_ +Default: `identity` + +Table that is used to translate email addresses before they +are matched against user_to_email values. + +Typically used to allow users to use their aliases as sender +addresses - prepare_email in this case should translate +aliases to "canonical" addresses. This is how it is +done in default configuration. + +If table does not contain any mapping for the used sender +address, it will be used as is. + +--- + +### check_header _boolean_ +Default: `yes` + +Whether to verify header sender in addition to envelope. + +Either Sender or From field value should match the +authorization identity. + +--- + +### unauth_action _action_ +Default: `reject` + +What to do if the user is not authenticated at all. + +--- + +### no_match_action _action_ +Default: `reject` + +What to do if user is not allowed to use the sender address specified. + +--- + +### malformed_action _action_ +Default: `reject` + +What to do if From or Sender header fields contain malformed values. + +--- + +### err_action _action_ +Default: `reject` + +What to do if error happens during prepare_email or user_to_email lookup. + +--- + +### auth_normalize _action_ +Default: `auto` + +Normalization function to apply to authorization username before +further processing. + +Available options: + +- `auto` `precis_casefold_email` for valid emails, `precis_casefold` otherwise. +- `precis_casefold_email` PRECIS UsernameCaseMapped profile + U-labels form for domain +- `precis_casefold` PRECIS UsernameCaseMapped profile for the entire string +- `precis_email` PRECIS UsernameCasePreserved profile + U-labels form for domain +- `precis` PRECIS UsernameCasePreserved profile for the entire string +- `casefold` Convert to lower case +- `noop` Nothing + +PRECIS profiles are defined by RFC 8265. In short, they make sure +that Unicode strings that look the same will be compared as if they were +the same. CaseMapped profiles also convert strings to lower case. + +--- + +### from_normalize _action_ +Default: `auto` + +Normalization function to apply to email addresses before +further processing. + +Available options are same as for `auth_normalize`. diff --git a/docs/reference/checks/command.md b/docs/reference/checks/command.md new file mode 100644 index 0000000..6475efc --- /dev/null +++ b/docs/reference/checks/command.md @@ -0,0 +1,96 @@ +# System command filter + +This module executes an arbitrary system command during a specified stage of +checks execution. + +``` +command executable_name arg0 arg1 ... { + run_on body + + code 1 reject + code 2 quarantine +} +``` + +## Arguments + +The module arguments specify the command to run. If the first argument is not +an absolute path, it is looked up in the Libexec Directory (/usr/lib/maddy on +Linux) and in $PATH (in that ordering). Note that no additional handling +of arguments is done, especially, the command is executed directly, not via the +system shell. + +There is a set of special strings that are replaced with the corresponding +message-specific values: + +- `{source_ip}` – IPv4/IPv6 address of the sending MTA. +- `{source_host}` – Hostname of the sending MTA, from the HELO/EHLO command. +- `{source_rdns}` – PTR record of the sending MTA IP address. +- `{msg_id}` – Internal message identifier. Unique for each delivery. +- `{auth_user}` – Client username, if authenticated using SASL PLAIN +- `{sender}` – Message sender address, as specified in the MAIL FROM SMTP command. +- `{rcpts}` – List of accepted recipient addresses, including the currently handled + one. +- `{address}` – Currently handled address. This is a recipient address if the command + is called during RCPT TO command handling (`run_on rcpt`) or a sender + address if the command is called during MAIL FROM command handling (`run_on + sender`). + +If value is undefined (e.g. `{source_ip}` for a message accepted over a Unix +socket) or unavailable (the command is executed too early), the placeholder +is replaced with an empty string. Note that it can not remove the argument. +E.g. `-i {source_ip}` will not become just `-i`, it will be `-i ""` + +Undefined placeholders are not replaced. + +## Command stdout + +The command stdout must be either empty or contain a valid RFC 5322 header. +If it contains a byte stream that does not look a valid header, the message +will be rejected with a temporary error. + +The header from stdout will be **prepended** to the message header. + +## Configuration directives + +### run_on `conn` | `sender` | `rcpt` | `body` +Default: `body` + +When to run the command. This directive also affects the information visible +for the message. + +- `conn`
+ Run before the sender address (MAIL FROM) is handled.
+ **Stdin**: Empty
+ **Available placeholders**: {source_ip}, {source_host}, {msg_id}, {auth_user}. + +- `sender`
+ Run during sender address (MAIL FROM) handling.
+ **Stdin**: Empty
+ **Available placeholders**: conn placeholders + {sender}, {address}. + The {address} placeholder contains the MAIL FROM address. + +- `rcpt`
+ Run during recipient address (RCPT TO) handling. The command is executed + once for each RCPT TO command, even if the same recipient is specified + multiple times.
+ **Stdin**: Empty
+ **Available placeholders**: sender placeholders + {rcpts}. + The {address} placeholder contains the recipient address. + +- `body`
+ Run during message body handling.
+ **Stdin**: The message header + body
+ **Available placeholders**: all except for {address}. + +--- + +### code _integer_ ignore
code _integer_ quarantine
code _integer_ reject _smtp-code_ _smtp-enhanced-code_ _smtp-message_ + +This directive specifies the mapping from the command exit code _integer_ to +the message pipeline action. + +Two codes are defined implicitly, exit code 1 causes the message to be rejected +with a permanent error, exit code 2 causes the message to be quarantined. Both +actions can be overridden using the 'code' directive. + diff --git a/docs/reference/checks/dkim.md b/docs/reference/checks/dkim.md new file mode 100644 index 0000000..7ab14a6 --- /dev/null +++ b/docs/reference/checks/dkim.md @@ -0,0 +1,63 @@ +# DKIM + +This is the check module that performs verification of the DKIM signatures +present on the incoming messages. + +## Configuration directives + +``` +check.dkim { + debug no + required_fields From Subject + allow_body_subset no + no_sig_action ignore + broken_sig_action ignore + fail_open no +} +``` + +### debug _boolean_ +Default: global directive value + +Log both successful and unsuccessful check executions instead of just +unsuccessful. + +--- + +### required_fields _string..._ +Default: `From Subject` + +Header fields that should be included in each signature. If signature +lacks any field listed in that directive, it will be considered invalid. + +Note that From is always required to be signed, even if it is not included in +this directive. + +--- + +### no_sig_action _action_ +Default: `ignore` (recommended by RFC 6376) + +Action to take when message without any signature is received. + +Note that DMARC policy of the sender domain can request more strict handling of +missing DKIM signatures. + +--- + +### broken_sig_action _action_ +Default: `ignore` (recommended by RFC 6376) + +Action to take when there are not valid signatures in a message. + +Note that DMARC policy of the sender domain can request more strict handling of +broken DKIM signatures. + +--- + +### fail_open _boolean_ +Default: `no` + +Whether to accept the message if a temporary error occurs during DKIM +verification. Rejecting the message with a 4xx code will require the sender +to resend it later in a hope that the problem will be resolved. diff --git a/docs/reference/checks/dnsbl.md b/docs/reference/checks/dnsbl.md new file mode 100644 index 0000000..a2d2736 --- /dev/null +++ b/docs/reference/checks/dnsbl.md @@ -0,0 +1,173 @@ +# DNSBL lookup + +The check.dnsbl module implements checking of source IP and hostnames against a set +of DNS-based Blackhole lists (DNSBLs). + +Its configuration consists of module configuration directives and a set +of blocks specifying lists to use and kind of lookups to perform on them. + +``` +check.dnsbl { + debug no + check_early no + + quarantine_threshold 1 + reject_threshold 1 + + # Lists configuration example. + dnsbl.example.org { + client_ipv4 yes + client_ipv6 no + ehlo no + mailfrom no + score 1 + } + hsrbl.example.org { + client_ipv4 no + client_ipv6 no + ehlo yes + mailfrom yes + score 1 + } +} +``` + +## Arguments + +Arguments specify the list of IP-based BLs to use. + +The following configurations are equivalent. + +``` +check { + dnsbl dnsbl.example.org dnsbl2.example.org +} +``` + +``` +check { + dnsbl { + dnsbl.example.org dnsbl2.example.org { + client_ipv4 yes + client_ipv6 no + ehlo no + mailfrom no + score 1 + } + } +} +``` + +## Configuration directives + +### debug _boolean_ +Default: global directive value + +Enable verbose logging. + +--- + +### check_early _boolean_ +Default: `no` + +Check BLs before mail delivery starts and silently reject blacklisted clients. + +For this to work correctly, check should not be used in source/destination +pipeline block. + +In particular, this means: + +- No logging is done for rejected messages. +- No action is taken if `quarantine_threshold` is hit, only `reject_threshold` + applies. +- `defer_sender_reject` from SMTP configuration takes no effect. +- MAIL FROM is not checked, even if specified. + +If you often get hit by spam attacks, it is recommended to enable this +setting to save server resources. + +--- + +### quarantine_threshold _integer_ +Default: `1` + +DNSBL score needed (equals-or-higher) to quarantine the message. + +--- + +### reject_threshold _integer_ +Default: `9999` + +DNSBL score needed (equals-or-higher) to reject the message. + +## List configuration + +``` +dnsbl.example.org dnsbl.example.com { + client_ipv4 yes + client_ipv6 no + ehlo no + mailfrom no + responses 127.0.0.1/24 + score 1 +} +``` + +Directive name and arguments specify the actual DNS zone to query when checking +the list. Using multiple arguments is equivalent to specifying the same +configuration separately for each list. + +### client_ipv4 _boolean_ +Default: `yes` + +Whether to check address of the IPv4 clients against the list. + +--- + +### client_ipv6 _boolean_ +Default: `yes` + +Whether to check address of the IPv6 clients against the list. + +--- + +### ehlo _boolean_ +Default: `no` + +Whether to check hostname specified n the HELO/EHLO command +against the list. + +This works correctly only with domain-based DNSBLs. + +--- + +### mailfrom _boolean_ +Default: `no` + +Whether to check domain part of the MAIL FROM address against the list. + +This works correctly only with domain-based DNSBLs. + +--- + +### responses _cidr_ | _ip..._ +Default: `127.0.0.1/24` + +IP networks (in CIDR notation) or addresses to permit in list lookup results. +Addresses not matching any entry in this directives will be ignored. + +--- + +### score _integer_ +Default: `1` + +Score value to add for the message if it is listed. + +If sum of list scores is equals or higher than `quarantine_threshold`, the +message will be quarantined. + +If sum of list scores is equals or higher than `rejected_threshold`, the message +will be rejected. + +It is possible to specify a negative value to make list act like a whitelist +and override results of other blocklists. diff --git a/docs/reference/checks/milter.md b/docs/reference/checks/milter.md new file mode 100644 index 0000000..8286a79 --- /dev/null +++ b/docs/reference/checks/milter.md @@ -0,0 +1,49 @@ +# Milter client + +The 'milter' implements subset of Sendmail's milter protocol that can be used +to integrate external software with maddy. +maddy implements version 6 of the protocol, older versions are +not supported. + +Notable limitations of protocol implementation in maddy include: +1. Changes of envelope sender address are not supported +2. Removal and addition of envelope recipients is not supported +3. Removal and replacement of header fields is not supported +4. Headers fields can be inserted only on top +5. Milter does not receive some "macros" provided by sendmail. + +Restrictions 1 and 2 are inherent to the maddy checks interface and cannot be +removed without major changes to it. Restrictions 3, 4 and 5 are temporary due to +incomplete implementation. + +``` +check.milter { + endpoint + fail_open false +} + +milter +``` + +## Arguments + +When defined inline, the first argument specifies endpoint to access milter +via. See below. + +## Configuration directives + +### endpoint _scheme://path_ +Default: not set + +Specifies milter protocol endpoint to use. +The endpoit is specified in standard URL-like format: +`tcp://127.0.0.1:6669` or `unix:///var/lib/milter/filter.sock` + +--- + +### fail_open _boolean_ +Default: `false` + +Toggles behavior on milter I/O errors. If false ("fail closed") - message is +rejected with temporary error code. If true ("fail open") - check is skipped. + diff --git a/docs/reference/checks/misc.md b/docs/reference/checks/misc.md new file mode 100644 index 0000000..19c71ad --- /dev/null +++ b/docs/reference/checks/misc.md @@ -0,0 +1,48 @@ +# Misc checks + +## Configuration directives + +Following directives are defined for all modules listed below. + +### fail_action `ignore` | `reject` | `quarantine` +Default: `quarantine` + +Action to take when check fails. See [Check actions](../actions/) for details. + +--- + +### debug _boolean_ +Default: global directive value + +Log both successful and unsuccessful check executions instead of just +unsuccessful. + +--- + +### require_mx_record + +Check that domain in MAIL FROM command does have a MX record and none of them +are "null" (contain a single dot as the host). + +By default, quarantines messages coming from servers missing MX records, +use `fail_action` directive to change that. + +--- + +### require_matching_rdns + +Check that source server IP does have a PTR record point to the domain +specified in EHLO/HELO command. + +By default, quarantines messages coming from servers with mismatched or missing +PTR record, use `fail_action` directive to change that. + +--- + +### require_tls + +Check that the source server is connected via TLS; either directly, or by using +the STARTTLS command. + +By default, rejects messages coming from unencrypted servers. Use the +`fail_action` directive to change that. \ No newline at end of file diff --git a/docs/reference/checks/rspamd.md b/docs/reference/checks/rspamd.md new file mode 100644 index 0000000..90063ae --- /dev/null +++ b/docs/reference/checks/rspamd.md @@ -0,0 +1,97 @@ +# rspamd + +The 'rspamd' module implements message filtering by contacting the rspamd +server via HTTP API. + +``` +check.rspamd { + tls_client { ... } + api_path http://127.0.0.1:11333 + settings_id whatever + tag maddy + hostname mx.example.org + io_error_action ignore + error_resp_action ignore + add_header_action quarantine + rewrite_subj_action quarantine + flags pass_all +} + +rspamd http://127.0.0.1:11333 +``` + +## Configuration directives + +### tls_client { ... } +Default: not set + +Configure TLS client if HTTPS is used. See [TLS configuration / Client](/reference/tls/#client) for details. + +--- + +### api_path _url_ +Default: `http://127.0.0.1:11333` + +URL of HTTP API endpoint. Supports both HTTP and HTTPS and can include +path element. + +--- + +### settings_id _string_ +Default: not set + +Settings ID to pass to the server. + +--- + +### tag _string_ +Default: `maddy` + +Value to send in MTA-Tag header field. + +--- + +### hostname _string_
+Default: value of global directive + +Value to send in MTA-Name header field. + +--- + +### io_error_action _action_ +Default: `ignore` + +Action to take in case of inability to contact the rspamd server. + +--- + +### error_resp_action _action_ +Default: `ignore` + +Action to take in case of 5xx or 4xx response received from the rspamd server. + +--- + +### add_header_action _action_ +Default: `quarantine` + +Action to take when rspamd requests to "add header". + +X-Spam-Flag and X-Spam-Score are added to the header irregardless of value. + +--- + +### rewrite_subj_action _action_ +Default: `quarantine` + +Action to take when rspamd requests to "rewrite subject". + +X-Spam-Flag and X-Spam-Score are added to the header irregardless of value. + +--- + +### flags _string-list..._ +Default: `pass_all` + +Flags to pass to the rspamd server. +See [https://rspamd.com/doc/architecture/protocol.html](https://rspamd.com/doc/architecture/protocol.html) for details. diff --git a/docs/reference/checks/spf.md b/docs/reference/checks/spf.md new file mode 100644 index 0000000..f0afb34 --- /dev/null +++ b/docs/reference/checks/spf.md @@ -0,0 +1,97 @@ +# SPF + +check.spf the check module that verifies whether IP address of the client is +authorized to send messages for domain in MAIL FROM address. + +SPF statuses are mapped to maddy check actions in a way +specified by \*_action directives. By default, SPF failure +results in the message being quarantined and errors (both permanent and +temporary) cause message to be rejected. +Authentication-Results field is generated irregardless of status. + +## DMARC override + +It is recommended by the DMARC standard to don't fail delivery based solely on +SPF policy and always check DMARC policy and take action based on it. + +If `enforce_early` is `no`, check.spf module will not take any action on SPF +policy failure if sender domain does have a DMARC record with 'quarantine' or +'reject' policy. Instead it will rely on DMARC support to take necesary +actions using SPF results as an input. + +Disabling `enforce_early` without enabling DMARC support will make SPF policies +no-op and is considered insecure. + +## Configuration directives + +``` +check.spf { + debug no + enforce_early no + fail_action quarantine + softfail_action ignore + permerr_action reject + temperr_action reject +} +``` + +### debug _boolean_ +Default: global directive value + +Enable verbose logging for check.spf. + +--- + +### enforce_early _boolean_ +Default: `no` + +Make policy decision on MAIL FROM stage (before the message body is received). +This makes it impossible to apply DMARC override (see above). + +--- + +### none_action `reject` | `quarantine` | `ignore` +Default: `ignore` + +Action to take when SPF policy evaluates to a 'none' result. + +See [https://tools.ietf.org/html/rfc7208#section-2.6](https://tools.ietf.org/html/rfc7208#section-2.6) for meaning of +SPF results. + +--- + +### neutral_action `reject` | `quarantine` | `ignore` +Default: `ignore` + +Action to take when SPF policy evaluates to a 'neutral' result. + +See [https://tools.ietf.org/html/rfc7208#section-2.6](https://tools.ietf.org/html/rfc7208#section-2.6) for meaning of +SPF results. + +--- + +### fail_action `reject` | `quarantine` | `ignore` +Default: `quarantine` + +Action to take when SPF policy evaluates to a 'fail' result. + +--- + +### softfail_action `reject` | `quarantine` | `ignore` +Default: `ignore` + +Action to take when SPF policy evaluates to a 'softfail' result. + +--- + +### permerr_action `reject` | `quarantine` | `ignore` +Default: `reject` + +Action to take when SPF policy evaluates to a 'permerror' result. + +--- + +### temperr_action `reject` | `quarantine` | `ignore` +Default: `reject` + +Action to take when SPF policy evaluates to a 'temperror' result. diff --git a/docs/reference/config-syntax.md b/docs/reference/config-syntax.md new file mode 100644 index 0000000..72e18d4 --- /dev/null +++ b/docs/reference/config-syntax.md @@ -0,0 +1,200 @@ +# Configuration files syntax + +**Note:** This file is a technical document describing how +maddy parses configuration files. + +Configuration consists of newline-delimited "directives". Each directive can +have zero or more arguments. + +``` +directive0 +directive1 arg0 arg1 +``` + +Any line starting with # is ignored. Empty lines are ignored too. + +## Quoting + +Strings with whitespace should be wrapped into double quotes to make sure they +will be interpreted as a single argument. + +``` +directive0 two arguments +directive1 "one argument" +``` + +String wrapped in quotes may contain newlines and they will not be interpreted +as a directive separator. + +``` +directive0 "one long big +argument for directive0" +``` + +Quotes and only quotes can be escaped inside literals: \\" + +Backslash can be used at the end of line to continue the directve on the next +line. + +## Blocks + +A directive may have several subdirectives. They are written in a {-enclosed +block like this: +``` +directive0 arg0 arg1 { + subdirective0 arg0 arg1 + subdirective1 etc +} +``` + +Subdirectives can have blocks too. + +``` +directive0 { + subdirective0 { + subdirective2 { + a + b + c + } + } + subdirective1 { } +} +``` + +Level of nesting is limited, but you should never hit the limit with correct +configuration. + +In most cases, an empty block is equivalent to no block: +``` +directive { } +directive2 # same as above +``` + +## Environment variables + +Environment variables can be referenced in the configuration using either +{env:VARIABLENAME} syntax. + +Non-existent variables are expanded to empty strings and not removed from +the arguments list. In the following example, directive0 will have one argument +independently of whether VAR is defined. + +``` +directive0 {env:VAR} +``` + +Parse is forgiving and incomplete variable placeholder (e.g. '{env:VAR') will +be left as-is. Variables are expanded inside quotes too. + +## Snippets & imports + +You can reuse blocks of configuration by defining them as "snippets". Snippet +is just a directive with a block, declared tp top level (not inside any blocks) +and with a directive name wrapped in curly braces. + +``` +(snippetname) { + a + b + c +} +``` + +The snippet can then be referenced using 'import' meta-directive. + +``` +unrelated0 +unrelated1 +import snippetname +``` + +The above example will be expanded into the following configuration: + +``` +unrelated0 +unrelated1 +a +b +c +``` + +Import statement also can be used to include content from other files. It works +exactly the same way as with snippets but the file path should be used instead. +The path can be either relative to the location of the currently processed +configuration file or absolute. If there are both snippet and file with the +same name - snippet will be used. + +``` +# /etc/maddy/tls.conf +tls long_path_to_certificate long_path_to_private_key + +# /etc/maddy/maddy.conf +smtp tcp://0.0.0.0:25 { + import tls.conf +} +``` + +``` +# Expanded into: +smtp tcp://0.0.0.0:25 { + tls long_path_to_certificate long_path_to_private_key +} +``` + +The imported file can introduce new snippets and they can be referenced in any +processed configuration file. + +## Duration values + +Directives that accept duration use the following format: A sequence of decimal +digits with an optional fraction and unit suffix (zero can be specified without +a suffix). If multiple values are specified, they will be added. + +Valid unit suffixes: "h" (hours), "m" (minutes), "s" (seconds), "ms" (milliseconds). +Implementation also accepts us and ns for microseconds and nanoseconds, but these +values are useless in practice. + +Examples: +``` +1h +1h 5m +1h5m +0 +``` + +## Data size values + +Similar to duration values, but fractions are not allowed and suffixes are different. + +Valid unit suffixes: "G" (gibibyte, 1024^3 bytes), "M" (mebibyte, 1024^2 bytes), +"K" (kibibyte, 1024 bytes), "B" or "b" (byte). + +Examples: +``` +32M +3M 5K +5b +``` + +Also note that the following is not valid, unlike Duration values syntax: +``` +32M5K +``` + +## Address Definitions + +Maddy configuration uses URL-like syntax to specify network addresses. + +- `unix://file_path` – Unix domain socket. Relative paths are relative to runtime directory (`/run/maddy`). +- `tcp://ADDRESS:PORT` – TCP/IP socket. +- `tls://ADDRESS:PORT` – TCP/IP socket using TLS. + +## Dummy Module + +No-op module. It doesn't need to be configured explicitly and can be referenced +using "dummy" name. It can act as a delivery target or auth. +provider. In the latter case, it will accept any credentials, allowing any +client to authenticate using any username and password (use with care!). + + diff --git a/docs/reference/endpoints/imap.md b/docs/reference/endpoints/imap.md new file mode 100644 index 0000000..ec0d2e6 --- /dev/null +++ b/docs/reference/endpoints/imap.md @@ -0,0 +1,164 @@ +# IMAP4rev1 endpoint + +Module 'imap' is a listener that implements IMAP4rev1 protocol and provides +access to local messages storage specified by 'storage' directive. + +In most cases, local storage modules will auto-create accounts when they are +accessed via IMAP. This relies on authentication provider used by IMAP endpoint +to provide what essentially is access control. There is a caveat, however: this +auto-creation will not happen when delivering incoming messages via SMTP as +there is no authentication to confirm that this account should indeed be +created. + +## Configuration directives + +``` +imap tcp://0.0.0.0:143 tls://0.0.0.0:993 { + tls /etc/ssl/private/cert.pem /etc/ssl/private/pkey.key + io_debug no + debug no + insecure_auth no + sasl_login no + auth pam + storage &local_mailboxes + auth_map identity + auth_map_normalize auto + storage_map identity + storage_map_normalize auto +} +``` + +### tls _certificate-path_ _key-path_ { ... } +Default: global directive value + +TLS certificate & key to use. Fine-tuning of other TLS properties is possible +by specifying a configuration block and options inside it: + +``` +tls cert.crt key.key { + protocols tls1.2 tls1.3 +} +``` + +See [TLS configuration / Server](/reference/tls/#server-side) for details. + +--- + +### proxy_protocol _trusted ips..._ { ... } +Default: not enabled + +Enable use of HAProxy PROXY protocol. Supports both v1 and v2 protocols. +If a list of trusted IP addresses or subnets is provided, only connections +from those will be trusted. + +TLS for the channel between the proxies and maddy can be configured +using a 'tls' directive: +``` +proxy_protocol { + trust 127.0.0.1 ::1 192.168.0.1/24 + tls &proxy_tls +} +``` +Note that the top-level 'tls' directive is not inherited here. If you +need TLS on top of the PROXY protocol, securing the protocol header, +you must declare TLS explicitly. + +--- + +### io_debug _boolean_ +Default: `no` + +Write all commands and responses to stderr. + +--- + +### io_errors _boolean_ +Default: `no` + +Log I/O errors. + +--- + +### debug _boolean_ +Default: global directive value + +Enable verbose logging. + +--- + +### insecure_auth _boolean_ +Default: `no` (`yes` if TLS is disabled) + +Allow plain-text authentication over unencrypted connections. + +--- + +### sasl_login _boolean_ +Default: `no` + +Enable support for SASL LOGIN authentication mechanism used by +some outdated clients. + +--- + +### auth _module-reference_ +**Required.** + +Use the specified module for authentication. + +--- + +### storage _module-reference_ +**Required.** + +Use the specified module for message storage. + +--- + +### storage_map _module-reference_ +Default: `identity` + +Use the specified table to map SASL usernames to storage account names. + +Before username is looked up, it is normalized using function defined by +`storage_map_normalize`. + +This directive is useful if you want users user@example.org and user@example.com +to share the same storage account named "user". In this case, use + +``` + storage_map email_localpart +``` + +Note that `storage_map` does not affect the username passed to the +authentication provider. + +It also does not affect how message delivery is handled, you should specify +`delivery_map` in storage module to define how to map email addresses +to storage accounts. E.g. + +``` + storage.imapsql local_mailboxes { + ... + delivery_map email_localpart # deliver "user@*" to mailbox for "user" + } +``` + +--- + +### storage_map_normalize _function_ +Default: `auto` + +Same as `auth_map_normalize` but for `storage_map`. + +--- + +### auth_map_normalize _function_ +Default: `auto` + +Overrides global `auth_map_normalize` value for this endpoint. + +See [Global configuration](/reference/global-config) for details. + + + diff --git a/docs/reference/endpoints/openmetrics.md b/docs/reference/endpoints/openmetrics.md new file mode 100644 index 0000000..f455f71 --- /dev/null +++ b/docs/reference/endpoints/openmetrics.md @@ -0,0 +1,41 @@ +# OpenMetrics/Prometheus telemetry + +Various server statistics are provided in OpenMetrics format by the +"openmetrics" module. + +To enable it, add the following line to the server config: + +``` +openmetrics tcp://127.0.0.1:9749 { } +``` + +Scrape endpoint would be `http://127.0.0.1:9749/metrics`. + +## Metrics + +``` +# AUTH command failures due to invalid credentials. +maddy_smtp_failed_logins{module} +# Failed SMTP transaction commands (MAIL, RCPT, DATA). +maddy_smtp_failed_commands{module, command, smtp_code, smtp_enchcode} +# Messages rejected with 4xx code due to ratelimiting. +maddy_smtp_ratelimit_deferred{module} +# Amount of started SMTP transactions started. +maddy_smtp_started_transactions{module} +# Amount of aborted SMTP transactions started. +maddy_smtp_aborted_transactions{module} +# Amount of completed SMTP transactions. +maddy_smtp_completed_transactions{module} +# Number of times a check returned 'reject' result (may be more than processed +# messages if check does so on per-recipient basis). +maddy_check_reject{check} +# Number of times a check returned 'quarantine' result (may be more than +# processed messages if check does so on per-recipient basis). +maddy_check_quarantined{check} +# Amount of queued messages. +maddy_queue_length{module, location} +# Outbound connections established with specific TLS security level. +maddy_remote_conns_tls_level{module, level} +# Outbound connections established with specific MX security level. +maddy_remote_conns_mx_level{module, level} +``` diff --git a/docs/reference/endpoints/smtp.md b/docs/reference/endpoints/smtp.md new file mode 100644 index 0000000..4dfa723 --- /dev/null +++ b/docs/reference/endpoints/smtp.md @@ -0,0 +1,312 @@ +# SMTP/LMTP/Submission endpoint + +Module 'smtp' is a listener that implements ESMTP protocol with optional +authentication, LMTP and Submission support. Incoming messages are processed in +accordance with pipeline rules (explained in Message pipeline section below). + +``` +smtp tcp://0.0.0.0:25 { + hostname example.org + tls /etc/ssl/private/cert.pem /etc/ssl/private/pkey.key + io_debug no + debug no + insecure_auth no + sasl_login no + read_timeout 10m + write_timeout 1m + max_message_size 32M + max_header_size 1M + auth pam + defer_sender_reject yes + dmarc yes + smtp_max_line_length 4000 + limits { + endpoint rate 10 + endpoint concurrency 500 + } + + # Example pipeline ocnfiguration. + destination example.org { + deliver_to &local_mailboxes + } + default_destination { + reject + } +} +``` + +## Configuration directives + +### hostname _string_ +Default: global directive value + +Server name to use in SMTP banner. + +``` +220 example.org ESMTP Service Ready +``` + +--- + +### tls _certificate-path_ _key-path_ { ... } +Default: global directive value + +TLS certificate & key to use. Fine-tuning of other TLS properties is possible +by specifying a configuration block and options inside it: + +``` +tls cert.crt key.key { + protocols tls1.2 tls1.3 +} +``` + +See [TLS configuration / Server](/reference/tls/#server-side) for details. + +--- + +### proxy_protocol _trusted ips..._ { ... }
+Default: not enabled + +Enable use of HAProxy PROXY protocol. Supports both v1 and v2 protocols. +If a list of trusted IP addresses or subnets is provided, only connections +from those will be trusted. + +TLS for the channel between the proxies and maddy can be configured +using a 'tls' directive: +``` +proxy_protocol { + trust 127.0.0.1 ::1 192.168.0.1/24 + tls &proxy_tls +} +``` + +--- + +### io_debug _boolean_ +Default: `no` + +Write all commands and responses to stderr. + +--- + +### debug _boolean_ +Default: global directive value + +Enable verbose logging. + +--- + +### insecure_auth _boolean_ +Default: `no` (`yes` if TLS is disabled) + +Allow plain-text authentication over unencrypted connections. Not recommended! + +--- + +### sasl_login _boolean_ +Default: `no` + +Enable support for SASL LOGIN authentication mechanism used by +some outdated clients. + +--- + +### read_timeout _duration_ +Default: `10m` + +I/O read timeout. + +--- + +### write_timeout _duration_ +Default: `1m` + +I/O write timeout. + +--- + +### max_message_size _size_ +Default: `32M` + +Limit the size of incoming messages to 'size'. + +--- + +### max_header_size _size_ +Default: `1M` + +Limit the size of incoming message headers to 'size'. + +--- + +### auth _module-reference_ +Default: not specified + +Use the specified module for authentication. + +--- + +### defer_sender_reject _boolean_ +Default: `yes` + +Apply sender-based checks and routing logic when first RCPT TO command +is received. This allows maddy to log recipient address of the rejected +message and also improves interoperability with (improperly implemented) +clients that don't expect an error early in session. + +--- + +### max_logged_rcpt_errors _integer_ +Default: `5` + +Amount of RCPT-time errors that should be logged. Further errors will be +handled silently. This is to prevent log flooding during email dictionary +attacks (address probing). + +--- + +### max_received _integer_ +Default: `50` + +Max. amount of Received header fields in the message header. If the incoming +message has more fields than this number, it will be rejected with the permanent error +5.4.6 ("Routing loop detected"). + +--- + +### buffer `ram`
buffer `fs` _path_
buffer `auto` _max-size_ _path_ +Default: `auto 1M StateDirectory/buffer` + +Temporary storage to use for the body of accepted messages. + +- `ram` – Store the body in RAM. +- `fs` – Write out the message to the FS and read it back as needed. +_path_ can be omitted and defaults to StateDirectory/buffer. +- `auto` – Store message bodies smaller than `_max_size_` entirely in RAM, +otherwise write them out to the FS. _path_ can be omitted and defaults to `StateDirectory/buffer`. + +--- + +### smtp_max_line_length _integer_ +Default: `4000` + +The maximum line length allowed in the SMTP input stream. If client sends a +longer line - connection will be closed and message (if any) will be rejected +with a permanent error. + +RFC 5321 has the recommended limit of 998 bytes. Servers are not required +to handle longer lines correctly but some senders may produce them. + +Unless BDAT extension is used by the sender, this limitation also applies to +the message body. + +--- + +### dmarc _boolean_ +Default: `yes` + +Enforce sender's DMARC policy. Due to implementation limitations, it is not a +check module. + +**Note**: Report generation is not implemented now. + +**Note**: DMARC needs SPF and DKIM checks to function correctly. +Without these, DMARC check will not run. + +--- + +## Rate & concurrency limiting + +### limits { ... } +Default: no limits + +This allows configuring a set of message flow restrictions including +max. concurrency and rate per-endpoint, per-source, per-destination. + +Limits are specified as directives inside the block: + +``` +limits { + all rate 20 + destination concurrency 5 +} +``` + +Supported limits: + +### _scope_ rate _burst_ _period_ + +Rate limit. Restrict the amount of messages processed in _period_ to +_burst_ messages. If period is not specified, 1 second is used. + +### _scope_ concurrency _max_ +Concurrency limit. Restrict the amount of messages processed in parallel +to _max_. + +For each supported limitation, _scope_ determines whether it should be applied +for all messages ("all"), per-sender IP ("ip"), per-sender domain ("source") or +per-recipient domain ("destination"). Having a scope other than "all" means +that the restriction will be enforced independently for each group determined +by scope. E.g. "ip rate 20" means that the same IP cannot send more than 20 +messages per second. "destination concurrency 5" means that no more than 5 +messages can be sent in parallel to a single domain. + +**Note**: At the moment, SMTP endpoint on its own does not support per-recipient +limits. They will be no-op. If you want to enforce a per-recipient restriction +on outbound messages, do so using 'limits' directive for the 'table.remote' module + +It is possible to share limit counters between multiple endpoints (or any other +modules). To do so define a top-level configuration block for module "limits" +and reference it where needed using standard & syntax. E.g. + +``` +limits inbound_limits { + all rate 20 +} + +smtp smtp://0.0.0.0:25 { + limits &inbound_limits + ... +} + +submission tls://0.0.0.0:465 { + limits &inbound_limits + ... +} +``` + +Using an "all rate" restriction in such way means that no more than 20 +messages can enter the server through both endpoints in one second. + +# Submission module (submission) + +Module 'submission' implements all functionality of the 'smtp' module and adds +certain message preprocessing on top of it, additionally authentication is +always required. + +'submission' module checks whether addresses in header fields From, Sender, To, +Cc, Bcc, Reply-To are correct and adds Message-ID and Date if it is missing. + +``` +submission tcp://0.0.0.0:587 tls://0.0.0.0:465 { + # ... same as smtp ... +} +``` + +# LMTP module (lmtp) + +Module 'lmtp' implements all functionality of the 'smtp' module but uses +LMTP (RFC 2033) protocol. + +``` +lmtp unix://lmtp.sock { + # ... same as smtp ... +} +``` + +## Limitations of LMTP implementation + +- Can't be used with TCP. +- Delivery to 'sql' module storage is always atomic, either all recipients will + succeed or none of them will. + diff --git a/docs/reference/global-config.md b/docs/reference/global-config.md new file mode 100644 index 0000000..db0ec1a --- /dev/null +++ b/docs/reference/global-config.md @@ -0,0 +1,153 @@ +# Global configuration directives + +These directives can be specified outside of any +configuration blocks and they are applied to all modules. + +Some directives can be overridden on per-module basis (e.g. hostname). + +### state_dir _path_ +Default: `/var/lib/maddy` + +The path to the state directory. This directory will be used to store all +persistent data and should be writable. + +--- + +### runtime_dir _path_ +Default: `/run/maddy` + +The path to the runtime directory. Used for Unix sockets and other temporary +objects. Should be writable. + +--- + +### hostname _domain_ +Default: not specified + +Internet hostname of this mail server. Typicall FQDN is used. It is recommended +to make sure domain specified here resolved to the public IP of the server. + +--- + +### auth_map _module-reference_ +Default: `identity` + +Use the specified table to translate SASL usernames before passing it to the +authentication provider. + +Before username is looked up, it is normalized using function defined by +`auth_map_normalize`. + +Note that `auth_map` does not affect the storage account name used. You probably +should also use `storage_map` in IMAP config block to handle this. + +This directive is useful if used authentication provider does not support +using emails as usernames but you still want users to have separate mailboxes +on separate domains. In this case, use it with `email_localpart` table: + +``` + auth_map email_localpart +``` + +With this configuration, `user@example.org` and `user@example.com` will use +`user` credentials when authenticating, but will access `user@example.org` and +`user@example.com` mailboxes correspondingly. If you want to also accept +`user` as a username, use `auth_map email_localpart_optional`. + +If you want `user@example.org` and `user@example.com` to have the same mailbox, +also set `storage_map` in IMAP config block to use `email_localpart` +(or `email_localpart_optional` if you want to also accept just "user"): + +``` + storage_map email_localpart +``` + +In this case you will need to create storage accounts without domain part in +the name: + +``` +maddy imap-acct create user # instead of user@example.org +``` + +--- + +### auth_map_normalize _function_ +Default: `auto` + +Normalization function to apply to SASL usernames before mapping +them to storage accounts. + +Available options: + +- `auto` `precis_casefold_email` for valid emails, `precis_casefold` otherwise. +- `precis_casefold_email` PRECIS UsernameCaseMapped profile + U-labels form for domain +- `precis_casefold` PRECIS UsernameCaseMapped profile for the entire string +- `precis_email` PRECIS UsernameCasePreserved profile + U-labels form for domain +- `precis` PRECIS UsernameCasePreserved profile for the entire string +- `casefold` Convert to lower case +- `noop` Nothing + +--- + +### autogenerated_msg_domain _domain_ +Default: not specified + +Domain that is used in From field for auto-generated messages (such as Delivery +Status Notifications). + +--- + +### tls `file` _cert-file_ _pkey-file_ | _module-reference_ | `off` +Default: not specified + +Default TLS certificate to use for all endpoints. + +Must be present in either all endpoint modules configuration blocks or as +global directive. + +You can also specify other configuration options such as cipher suites and TLS +version. See maddy-tls(5) for details. maddy uses reasonable +cipher suites and TLS versions by default so you generally don't have to worry +about it. + +--- + +### tls_client { ... } +Default: not specified + +This is optional block that specifies various TLS-related options to use when +making outbound connections. See TLS client configuration for details on +directives that can be used in it. maddy uses reasonable cipher suites and TLS +versions by default so you generally don't have to worry about it. + +--- + +### log _targets..._ | `off` +Default: `stderr` + +Write log to one of more "targets". + +The target can be one or the following: + +- `stderr` – Write logs to stderr. +- `stderr_ts` – Write logs to stderr with timestamps. +- `syslog` – Send logs to the local syslog daemon. +- _file path_ – Write (append) logs to file. + +Example: + +``` +log syslog /var/log/maddy.log +``` + +**Note:** Maddy does not perform log files rotation, this is the job of the +logrotate daemon. Send SIGUSR1 to maddy process to make it reopen log files. + +--- + +### debug _boolean_ +Default: `no` + +Enable verbose logging for all modules. You don't need that unless you are +reporting a bug. + diff --git a/docs/reference/modifiers/dkim.md b/docs/reference/modifiers/dkim.md new file mode 100644 index 0000000..36fffe2 --- /dev/null +++ b/docs/reference/modifiers/dkim.md @@ -0,0 +1,225 @@ +# DKIM signing + +modify.dkim module is a modifier that signs messages using DKIM +protocol (RFC 6376). + +Each configuration block specifies a single selector +and one or more domains. + +A key will be generated or read for each domain, the key to use +for each message will be selected based on the SMTP envelope sender. Exception +for that is that for domain-less postmaster address and null address, the +key for the first domain will be used. If domain in envelope sender +does not match any of loaded keys, message will not be signed. +Additionally, for each messages From header is checked to +match MAIL FROM and authorization identity (username sender is logged in as). +This can be controlled using require_sender_match directive. + +Generated private keys are stored in unencrypted PKCS#8 format +in state_directory/dkim_keys (`/var/lib/maddy/dkim_keys`). +In the same directory .dns files are generated that contain +public key for each domain formatted in the form of a DNS record. + +## Arguments + +domains and selector can be specified in arguments, so actual modify.dkim use can +be shortened to the following: + +``` +modify { + dkim example.org selector +} +``` + +## Configuration directives + +``` +modify.dkim { + debug no + domains example.org example.com + selector default + key_path dkim-keys/{domain}-{selector}.key + oversign_fields ... + sign_fields ... + header_canon relaxed + body_canon relaxed + sig_expiry 120h # 5 days + hash sha256 + newkey_algo rsa2048 +} +``` + +### debug _boolean_ +Default: global directive value + +Enable verbose logging. + +--- + +### domains _string-list_ +**Required**.
+Default: not specified + + +ADministrative Management Domains (ADMDs) taking responsibility for messages. + +Should be specified either as a directive or as an argument. + +--- + +### selector _string_ +**Required**.
+Default: not specified + +Identifier of used key within the ADMD. +Should be specified either as a directive or as an argument. + +--- + +### key_path _string_ +Default: `dkim_keys/{domain}_{selector}.key` + +Path to private key. It should be in PKCS#8 format wrapped in PAM encoding. +If key does not exist, it will be generated using algorithm specified +in newkey_algo. + +Placeholders '{domain}' and '{selector}' will be replaced with corresponding +values from domain and selector directives. + +Additionally, keys in PKCS#1 ("RSA PRIVATE KEY") and +RFC 5915 ("EC PRIVATE KEY") can be read by modify.dkim. Note, however that +newly generated keys are always in PKCS#8. + +--- + +### oversign_fields _list..._ +Default: see below + +Header fields that should be signed n+1 times where n is times they are +present in the message. This makes it impossible to replace field +value by prepending another field with the same name to the message. + +Fields specified here don't have to be also specified in `sign_fields`. + +Default set of oversigned fields: + +- Subject +- To +- From +- Date +- MIME-Version +- Content-Type +- Content-Transfer-Encoding +- Reply-To +- Message-Id +- References +- Autocrypt +- Openpgp + +--- + +### sign_fields _list..._ +Default: see below + +Header fields that should be signed n times where n is times they are +present in the message. For these fields, additional values can be prepended +by intermediate relays, but existing values can't be changed. + +Default set of signed fields: + +- List-Id +- List-Help +- List-Unsubscribe +- List-Post +- List-Owner +- List-Archive +- Resent-To +- Resent-Sender +- Resent-Message-Id +- Resent-Date +- Resent-From +- Resent-Cc + +--- + +### header_canon `relaxed` | `simple` +Default: `relaxed` + +Canonicalization algorithm to use for header fields. With `relaxed`, whitespace within +fields can be modified without breaking the signature, with `simple` no +modifications are allowed. + +--- + +### body_canon `relaxed` | `simple` +Default: `relaxed` + +Canonicalization algorithm to use for message body. With `relaxed`, whitespace within +can be modified without breaking the signature, with `simple` no +modifications are allowed. + +--- + +### sig_expiry _duration_ +Default: `120h` + +Time for which signature should be considered valid. Mainly used to prevent +unauthorized resending of old messages. + +--- + +### hash _hash_ +Default: `sha256` + +Hash algorithm to use when computing body hash. + +sha256 is the only supported algorithm now. + +--- + +### newkey_algo `rsa4096` | `rsa2048` | `ed25519` +Default: `rsa2048` + +Algorithm to use when generating a new key. + +Currently ed25519 is **not** supported by most platforms. + +--- + +### require_sender_match _ids..._ +Default: `envelope auth` + +Require specified identifiers to match From header field and key domain, +otherwise - don't sign the message. + +If From field contains multiple addresses, message will not be +signed unless `allow_multiple_from` is also specified. In that +case only first address will be compared. + +Matching is done in a case-insensitive way. + +Valid values: + +- `off` – Disable check, always sign. +- `envelope` – Require MAIL FROM address to match From header. +- `auth` – If authorization identity contains @ - then require it to + fully match From header. Otherwise, check only local-part + (username). + +--- + +### allow_multiple_from _boolean_ +Default: `no` + +Allow multiple addresses in From header field for purposes of +`require_sender_match` checks. Only first address will be checked, however. + +--- + +### sign_subdomains _boolean_ +Default: `no` + +Sign emails from subdomains using a top domain key. + +Allows only one domain to be specified (can be worked around by using `modify.dkim` +multiple times). diff --git a/docs/reference/modifiers/envelope.md b/docs/reference/modifiers/envelope.md new file mode 100644 index 0000000..0e101cf --- /dev/null +++ b/docs/reference/modifiers/envelope.md @@ -0,0 +1,63 @@ +# Envelope sender / recipient rewriting + +`replace_sender` and `replace_rcpt` modules replace SMTP envelope addresses +based on the mapping defined by the table module (maddy-tables(5)). It is possible +to specify 1:N mappings. This allows, for example, implementing mailing lists. + +The address is normalized before lookup (Punycode in domain-part is decoded, +Unicode is normalized to NFC, the whole string is case-folded). + +First, the whole address is looked up. If there is no replacement, local-part +of the address is looked up separately and is replaced in the address while +keeping the domain part intact. Replacements are not applied recursively, that +is, lookup is not repeated for the replacement. + +Recipients are not deduplicated after expansion, so message may be delivered +multiple times to a single recipient. However, used delivery target can apply +such deduplication (imapsql storage does it). + +Definition: + +``` +replace_rcpt
[table arguments] { + [extended table config] +} +replace_sender
[table arguments] { + [extended table config] +} +``` + +Use examples: + +``` +modify { + replace_rcpt file /etc/maddy/aliases + replace_rcpt static { + entry a@example.org b@example.org + entry c@example.org c1@example.org c2@example.org + } + replace_rcpt regexp "(.+)@example.net" "$1@example.org" + replace_rcpt regexp "(.+)@example.net" "$1@example.org" "$1@example.com" +} +``` + +Possible contents of /etc/maddy/aliases in the example above: + +``` +# Replace 'cat' with any domain to 'dog'. +# E.g. cat@example.net -> dog@example.net +cat: dog + +# Replace cat@example.org with cat@example.com. +# Takes priority over the previous line. +cat@example.org: cat@example.com + +# Using aliases in multiple lines +cat2: dog +cat2: mouse +cat2@example.org: cat@example.com +cat2@example.org: cat@example.net +# Comma-separated aliases in multiple lines +cat3: dog , mouse +cat3@example.org: cat@example.com , cat@example.net +``` \ No newline at end of file diff --git a/docs/reference/modules.md b/docs/reference/modules.md new file mode 100644 index 0000000..f327e86 --- /dev/null +++ b/docs/reference/modules.md @@ -0,0 +1,76 @@ +# Modules introduction + +maddy is built of many small components called "modules". Each module does one +certain well-defined task. Modules can be connected to each other in arbitrary +ways to achieve wanted functionality. Default configuration file defines +set of modules that together implement typical email server stack. + +To specify the module that should be used by another module for something, look +for configuration directives with "module reference" argument. Then +put the module name as an argument for it. Optionally, if referenced module +needs that, put additional arguments after the name. You can also put a +configuration block with additional directives specifing the module +configuration. + +Here are some examples: + +``` +smtp ... { + # Deliver messages to the 'dummy' module with the default configuration. + deliver_to dummy + + # Deliver messages to the 'target.smtp' module with + # 'tcp://127.0.0.1:1125' argument as a configuration. + deliver_to smtp tcp://127.0.0.1:1125 + + # Deliver messages to the 'queue' module with the specified configuration. + deliver_to queue { + target ... + max_tries 10 + } +} +``` + +Additionally, module configuration can be placed in a separate named block +at the top-level and referenced by its name where it is needed. + +Here is the example: +``` +storage.imapsql local_mailboxes { + driver sqlite3 + dsn all.db +} + +smtp ... { + deliver_to &local_mailboxes +} +``` + +It is recommended to use this syntax for modules that are 'expensive' to +initialize such as storage backends and authentication providers. + +For top-level configuration block definition, syntax is as follows: +``` +namespace.module_name config_block_name... { + module_configuration +} +``` +If config\_block\_name is omitted, it will be the same as module\_name. Multiple +names can be specified. All names must be unique. + +Note the "storage." prefix. This is the actual module name and includes +"namespace". It is a little cheating to make more concise names and can +be omitted when you reference the module where it is used since it can +be implied (e.g. putting module reference in "check{}" likely means you want +something with "check." prefix) + +Usual module arguments can't be specified when using this syntax, however, +modules usually provide explicit directives that allow to specify the needed +values. For example 'sql sqlite3 all.db' is equivalent to +``` +storage.imapsql { + driver sqlite3 + dsn all.db +} +``` + diff --git a/docs/reference/smtp-pipeline.md b/docs/reference/smtp-pipeline.md new file mode 100644 index 0000000..b094343 --- /dev/null +++ b/docs/reference/smtp-pipeline.md @@ -0,0 +1,408 @@ +# SMTP message routing (pipeline) + +# Message pipeline + +A message pipeline is a set of module references and associated rules that +describe how to handle messages. + +The pipeline is responsible for + +- Running message filters (called "checks"), (e.g. DKIM signature verification, + DNSBL lookup, and so on). +- Running message modifiers (e.g. DKIM signature creation). +- Associating each message recipient with one or more delivery targets. + Delivery target is a module that does the final processing (delivery) of the + message. + +Message handling flow is as follows: + +- Execute checks referenced in top-level `check` blocks (if any) +- Execute modifiers referenced in top-level `modify` blocks (if any) +- If there are `source` blocks - select one that matches the message sender (as + specified in MAIL FROM). If there are no `source` blocks - the entire + configuration is assumed to be the `default_source` block. +- Execute checks referenced in `check` blocks inside the selected `source` block + (if any). +- Execute modifiers referenced in `modify` blocks inside selected `source` + block (if any). + +Then, for each recipient: + +- Select the `destination` block that matches it. If there are + no `destination` blocks - the entire used `source` block is interpreted as if it + was a `default_destination` block. +- Execute checks referenced in the `check` block inside the selected `destination` + block (if any). +- Execute modifiers referenced in `modify` block inside the selected `destination` + block (if any). +- If the used block contains the `reject` directive - reject the recipient with + the specified SMTP status code. +- If the used block contains the `deliver_to` directive - pass the message to the + specified target module. Only recipients that are handled + by the used block are visible to the target. + +Each recipient is handled only by a single `destination` block, in case of +overlapping `destination` - the first one takes priority. + +``` +destination example.org { + deliver_to targetA +} +destination example.org { # ambiguous and thus not allowed + deliver_to targetB +} +``` + +Same goes for `source` blocks, each message is handled only by a single block. + +Each recipient block should contain at least one `deliver_to` directive or +`reject` directive. If `destination` blocks are used, then +`default_destination` block should also be used to specify behavior for +unmatched recipients. Same goes for source blocks, `default_source` should be +used if `source` is used. + +That is, pipeline configuration should explicitly specify behavior for each +possible sender/recipient combination. + +Additionally, directives that specify final handling decision (`deliver_to`, +`reject`) can't be used at the same level as source/destination rules. +Consider example: + +``` +destination example.org { + deliver_to local_mboxes +} +reject +``` + +It is not obvious whether `reject` applies to all recipients or +just for non-example.org ones, hence this is not allowed. + +Complete configuration example using all of the mentioned directives: + +``` +check { + # Run a check to make sure source SMTP server identification + # is legit. + spf +} + +# Messages coming from senders at example.org will be handled in +# accordance with the following configuration block. +source example.org { + # We are example.com, so deliver all messages with recipients + # at example.com to our local mailboxes. + destination example.com { + deliver_to &local_mailboxes + } + + # We don't do anything with recipients at different domains + # because we are not an open relay, thus we reject them. + default_destination { + reject 521 5.0.0 "User not local" + } +} + +# We do our business only with example.org, so reject all +# other senders. +default_source { + reject +} +``` + +## Directives + + +### check _block name_ { ... } +Context: pipeline configuration, source block, destination block + +List of the module references for checks that should be executed on +messages handled by block where 'check' is placed in. + +Note that message body checks placed in destination block are currently +ignored. Due to the way SMTP protocol is defined, they would cause message to +be rejected for all recipients which is not what you usually want when using +such configurations. + +Example: + +``` +check { + # Reference implicitly defined default configuration for check. + spf + + # Inline definition of custom config. + spf { + # Configuration for spf goes here. + permerr_action reject + } +} +``` + +It is also possible to define the block of checks at the top level +as "checks" module and reference it using & syntax. Example: + +``` +checks inbound_checks { + spf + dkim +} + +# ... somewhere else ... +{ + ... + check &inbound_checks +} +``` + +--- + +### modify { ... } +Default: not specified
+Context: pipeline configuration, source block, destination block + +List of the module references for modifiers that should be executed on +messages handled by block where 'modify' is placed in. + +Message modifiers are similar to checks with the difference in that checks +purpose is to verify whether the message is legitimate and valid per local +policy, while modifier purpose is to post-process message and its metadata +before final delivery. + +For example, modifier can replace recipient address to make message delivered +to the different mailbox or it can cryptographically sign outgoing message +(e.g. using DKIM). Some modifier can perform multiple unrelated modifications +on the message. + +**Note**: Modifiers that affect source address can be used only globally or on +per-source basis, they will be no-op inside destination blocks. Modifiers that +affect the message header will affect it for all recipients. + +It is also possible to define the block of modifiers at the top level +as "modiifers" module and reference it using & syntax. Example: + +``` +modifiers local_modifiers { + replace_rcpt file /etc/maddy/aliases +} + +# ... somewhere else ... +{ + ... + modify &local_modifiers +} +``` + +--- + +### reject _smtp-code_ _smtp-enhanced-code_ _error-description_
reject _smtp-code_ _smtp-enhanced-code_
reject _smtp-code_
reject +Context: destination block + +Messages handled by the configuration block with this directive will be +rejected with the specified SMTP error. + +If you aren't sure which codes to use, use 541 and 5.4.0 with your message or +just leave all arguments out, the error description will say "message is +rejected due to policy reasons" which is usually what you want to mean. + +`reject` can't be used in the same block with `deliver_to` or +`destination`/`source` directives. + +Example: + +``` +reject 541 5.4.0 "We don't like example.org, go away" +``` + +--- + +### deliver_to _target-config-block_ +Context: pipeline configuration, source block, destination block + +Deliver the message to the referenced delivery target. What happens next is +defined solely by used target. If `deliver_to` is used inside `destination` +block, only matching recipients will be passed to the target. + +--- + +### source_in _table-reference_ { ... } +Context: pipeline configuration + +Handle messages with envelope senders present in the specified table in +accordance with the specified configuration block. + +Takes precedence over all `sender` directives. + +Example: + +``` +source_in file /etc/maddy/banned_addrs { + reject 550 5.7.0 "You are not welcome here" +} +source example.org { + ... +} +... +``` + +See `destination_in` documentation for note about table configuration. + +--- + +### source _rules..._ { ... } +Context: pipeline configuration + +Handle messages with MAIL FROM value (sender address) matching any of the rules +in accordance with the specified configuration block. + +"Rule" is either a domain or a complete address. In case of overlapping +'rules', first one takes priority. Matching is case-insensitive. + +Example: + +``` +# All messages coming from example.org domain will be delivered +# to local_mailboxes. +source example.org { + deliver_to &local_mailboxes +} +# Messages coming from different domains will be rejected. +default_source { + reject 521 5.0.0 "You were not invited" +} +``` + +--- + +### reroute { ... } +Context: pipeline configuration, source block, destination block + +This directive allows to make message routing decisions based on the +result of modifiers. The block can contain all pipeline directives and they +will be handled the same with the exception that source and destination rules +will use the final recipient and sender values (e.g. after all modifiers are +applied). + +Here is the concrete example how it can be useful: + +``` +destination example.org { + modify { + replace_rcpt file /etc/maddy/aliases + } + reroute { + destination example.org { + deliver_to &local_mailboxes + } + default_destination { + deliver_to &remote_queue + } + } +} +``` + +This configuration allows to specify alias local addresses to remote ones +without being an open relay, since remote_queue can be used only if remote +address was introduced as a result of rewrite of local address. + +**Warning**: If you have DMARC enabled (default), results generated by SPF +and DKIM checks inside a reroute block **will not** be considered in DMARC +evaluation. + +--- + +### destination_in _table-reference_ { ... } +Context: pipeline configuration, source block + +Handle messages with envelope recipients present in the specified table in +accordance with the specified configuration block. + +Takes precedence over all 'destination' directives. + +Example: + +``` +destination_in file /etc/maddy/remote_addrs { + deliver_to smtp tcp://10.0.0.7:25 +} +destination example.com { + deliver_to &local_mailboxes +} +... +``` + +Note that due to the syntax restrictions, it is not possible to specify +extended configuration for table module. E.g. this is not valid: + +``` +destination_in sql_table { + dsn ... + driver ... +} { + deliver_to whatever +} +``` + +In this case, configuration should be specified separately and be referneced +using '&' syntax: + +``` +table.sql_table remote_addrs { + dsn ... + driver ... +} + +whatever { + destination_in &remote_addrs { + deliver_to whatever + } +} +``` + +--- + +### destination _rule..._ { ... } +Context: pipeline configuration, source block + +Handle messages with RCPT TO value (recipient address) matching any of the +rules in accordance with the specified configuration block. + +"Rule" is either a domain or a complete address. Duplicate rules are not +allowed. Matching is case-insensitive. + +Note that messages with multiple recipients are split into multiple messages if +they have recipients matched by multiple blocks. Each block will see the +message only with recipients matched by its rules. + +Example: + +``` +# Messages with recipients at example.com domain will be +# delivered to local_mailboxes target. +destination example.com { + deliver_to &local_mailboxes +} + +# Messages with other recipients will be rejected. +default_destination { + rejected 541 5.0.0 "User not local" +} +``` + +## Reusable pipeline snippets (msgpipeline module) + +The message pipeline can be used independently of the SMTP module in other +contexts that require a delivery target via `msgpipeline` module. + +Example: + +``` +msgpipeline local_routing { + destination whatever.com { + deliver_to dummy + } +} + +# ... somewhere else ... +deliver_to &local_routing +``` \ No newline at end of file diff --git a/docs/reference/storage/imap-filters.md b/docs/reference/storage/imap-filters.md new file mode 100644 index 0000000..b125a07 --- /dev/null +++ b/docs/reference/storage/imap-filters.md @@ -0,0 +1,70 @@ +# IMAP filters + +Most storage backends support application of custom code late in delivery +process. As opposed to using SMTP pipeline modifiers or checks, it allows +modifying IMAP-specific message attributes. In particular, it allows +code to change target folder and add IMAP flags (keywords) to the message. + +There is no way to reject message using IMAP filters, this should be done +earlier in SMTP pipeline logic. Quarantined messages are not processed +by IMAP filters and are unconditionally delivered to Junk folder (or other +folder with \Junk special-use attribute). + +To use an IMAP filter, specify it in the 'imap\_filter' directive for the +used storage backend, like this: +``` +storage.imapsql local_mailboxes { + ... + + imap_filter { + command /etc/maddy/sieve.sh {account_name} + } +} +``` + +## System command filter (imap.filter.command) + +This filter is similar to check.command module +and runs a system command to obtain necessary information. + +Usage: +``` +command executable_name args... { } +``` + +Same as check.command, following placeholders are supported for command +arguments: {source\_ip}, {source\_host}, {source\_rdns}, {msg\_id}, {auth\_user}, +{sender}. Note: placeholders +in command name are not processed to avoid possible command injection attacks. + +Additionally, for imap.filter.command, {account\_name} placeholder is replaced +with effective IMAP account name, {rcpt_to}, {original_rcpt_to} provide +access to the SMTP envelope recipient (before and after any rewrites), +{subject} is replaced with the Subject header, if it is present. + +Note that if you use provided systemd units on Linux, maddy executable is +sandboxed - all commands will be executed with heavily restricted filesystem +access and other privileges. Notably, /tmp is isolated and all directories +except for /var/lib/maddy and /run/maddy are read-only. You will need to modify +systemd unit if your command needs more privileges. + +Command output should consist of zero or more lines. First one, if non-empty, overrides +destination folder. All other lines contain additional IMAP flags to add +to the message. If command wants to add flags without changing folder - first +line should be empty. + +It is valid for command to not write anything to stdout. In this case its +execution will have no effect on delivery. + +Output example: +``` +Junk +``` +In this case, message will be placed in the Junk folder. + +``` + +$Label1 +``` +In this case, message will be placed in inbox and will have +'$Label1' added. diff --git a/docs/reference/storage/imapsql.md b/docs/reference/storage/imapsql.md new file mode 100644 index 0000000..f1abbb3 --- /dev/null +++ b/docs/reference/storage/imapsql.md @@ -0,0 +1,208 @@ +# SQL-indexed storage + +The imapsql module implements database for IMAP index and message +metadata using SQL-based relational database. + +Message contents are stored in an "blob store" defined by msg_store +directive. By default this is a file system directory under /var/lib/maddy. + +Supported RDBMS: +- SQLite 3.25.0 +- PostgreSQL 9.6 or newer +- CockroachDB 20.1.5 or newer + +Account names are required to have the form of a email address (unless configured otherwise) +and are case-insensitive. UTF-8 names are supported with restrictions defined in the +PRECIS UsernameCaseMapped profile. + +``` +storage.imapsql { + driver sqlite3 + dsn imapsql.db + msg_store fs messages/ +} +``` + +imapsql module also can be used as a lookup table. +It returns empty string values for existing usernames. This might be useful +with `destination_in` directive e.g. to implement catch-all +addresses (this is a bad idea to do so, this is just an example): +``` +destination_in &local_mailboxes { + deliver_to &local_mailboxes +} +destination example.org { + modify { + replace_rcpt regexp ".*" "catchall@example.org" + } + deliver_to &local_mailboxes +} +``` + + +## Arguments + +Specify the driver and DSN. + +## Configuration directives + +### driver _string_ +**Required.**
+Default: not specified + +Use a specified driver to communicate with the database. Supported values: +sqlite3, postgres. + +Should be specified either via an argument or via this directive. + +--- + +### dsn _string_ +**Required.**
+Default: not specified + +Data Source Name, the driver-specific value that specifies the database to use. + +For SQLite3 this is just a file path. +For PostgreSQL: [https://godoc.org/github.com/lib/pq#hdr-Connection\_String\_Parameters](https://godoc.org/github.com/lib/pq#hdr-Connection\_String\_Parameters) + +Should be specified either via an argument or via this directive. + +--- + +### msg_store _store_ +Default: `fs messages/` + +Module to use for message bodies storage. + +See "Blob storage" section for what you can use here. + +--- + +### compression `off`
compression _algorithm_
compression _algorithm_ _level_ +Default: `off` + +Apply compression to message contents. +Supported algorithms: `lz4`, `zstd`. + +--- + +### appendlimit _size_ +Default: `32M` + +Don't allow users to add new messages larger than 'size'. + +This does not affect messages added when using module as a delivery target. +Use `max_message_size` directive in SMTP endpoint module to restrict it too. + +--- + +### debug _boolean_ +Default: global directive value + +Enable verbose logging. + +--- + +### junk_mailbox _name_ +Default: `Junk` + +The folder to put quarantined messages in. Thishis setting is not used if user +does have a folder with "Junk" special-use attribute. + +--- + +### disable_recent _boolean_ +Default: `true` + +Disable RFC 3501-conforming handling of \Recent flag. + +This significantly improves storage performance when SQLite3 or CockroackDB is +used at the cost of confusing clients that use this flag. + +--- + +### sqlite_cache_size _integer_ +Default: defined by SQLite + +SQLite page cache size. If positive - specifies amount of pages (1 page - 4 +KiB) to keep in cache. If negative - specifies approximate upper bound +of cache size in KiB. + +--- + +### sqlite_busy_timeout _integer_ +Default: `5000000` + +SQLite-specific performance tuning option. Amount of milliseconds to wait +before giving up on DB lock. + +--- + +### imap_filter { ... } +Default: not set + +Specifies IMAP filters to apply for messages delivered from SMTP pipeline. + +Ex. + +``` +imap_filter { + command /etc/maddy/sieve.sh {account_name} +} +``` + +--- + +### delivery_map _table_ +Default: `identity` + +Use specified table module to map recipient +addresses from incoming messages to mailbox names. + +Normalization algorithm specified in `delivery_normalize` is appied before +`delivery_map`. + +--- + +### delivery_normalize _name_ +Default: `precis_casefold_email` + +Normalization function to apply to email addresses before mapping them +to mailboxes. + +See `auth_normalize`. + +--- + +### auth_map _table_ +**Deprecated:** Use `storage_map` in imap config instead.
+Default: `identity` + +Use specified table module to map authentication +usernames to mailbox names. + +Normalization algorithm specified in auth_normalize is applied before +auth_map. + +--- + +### auth_normalize _name_ +**Deprecated:** Use `storage_map_normalize` in imap config instead.
+**Default**: `precis_casefold_email` + +Normalization function to apply to authentication usernames before mapping +them to mailboxes. + +Available options: + +- `precis_casefold_email` PRECIS UsernameCaseMapped profile + U-labels form for domain +- `precis_casefold` PRECIS UsernameCaseMapped profile for the entire string +- `precis_email` PRECIS UsernameCasePreserved profile + U-labels form for domain +- `precis` PRECIS UsernameCasePreserved profile for the entire string +- `casefold` Convert to lower case +- `noop` Nothing + +Note: On message delivery, recipient address is unconditionally normalized +using `precis_casefold_email` function. + diff --git a/docs/reference/table/auth.md b/docs/reference/table/auth.md new file mode 100644 index 0000000..4bfe4bd --- /dev/null +++ b/docs/reference/table/auth.md @@ -0,0 +1,6 @@ +# Authentication providers + +Most authentication providers are also usable as a table +that contains all usernames known to the module. Exceptions are auth.external and +pam as underlying interfaces do not define a way to check credentials +existence. diff --git a/docs/reference/table/chain.md b/docs/reference/table/chain.md new file mode 100644 index 0000000..1cbc24c --- /dev/null +++ b/docs/reference/table/chain.md @@ -0,0 +1,41 @@ +# Table chaining + +The table.chain module allows chaining together multiple table modules +by using value returned by a previous table as an input for the second +table. + +Example: +``` +table.chain { + step regexp "(.+)(\\+[^+"@]+)?@example.org" "$1@example.org" + step file /etc/maddy/emails +} +``` +This will strip +prefix from mailbox before looking it up +in /etc/maddy/emails list. + +## Configuration directives + +### step _table_ + +Adds a table module to the chain. If input value is not in the table +(e.g. file) - return "not exists" error. + +--- + +### optional_step _table_ + +Same as step but if input value is not in the table - it is passed to the +next step without changes. + +Example: +Something like this can be used to map emails to usernames +after translating them via aliases map: + +``` +table.chain { + optional_step file /etc/maddy/aliases + step regexp "(.+)@(.+)" "$1" +} +``` + diff --git a/docs/reference/table/email_localpart.md b/docs/reference/table/email_localpart.md new file mode 100644 index 0000000..19b90f1 --- /dev/null +++ b/docs/reference/table/email_localpart.md @@ -0,0 +1,20 @@ +# Email local part + +The module `table.email_localpart` extracts and unescapes local ("username") part +of the email address. + +E.g. + +* `test@example.org` => `test` +* `"test @ a"@example.org` => `test @ a` + +Mappings for invalid emails are not defined (will be treated as non-existing +values). + +``` +table.email_localpart { } +``` + +`table.email_localpart_optional` works the same, but returns non-email strings +as is. This can be used if you want to accept both `user@example.org` and +`user` somewhere and treat it the same. diff --git a/docs/reference/table/email_with_domain.md b/docs/reference/table/email_with_domain.md new file mode 100644 index 0000000..6a719e0 --- /dev/null +++ b/docs/reference/table/email_with_domain.md @@ -0,0 +1,37 @@ +# Email with domain + +The table module `table.email_with_domain` appends one or more +domains (allowing 1:N expansion) to the specified value. + +``` +table.email_with_domain DOMAIN DOMAIN... { } +``` + +It can be used to implement domain-level expansion for aliases if used together +with `table.chain`. Example: + +``` +modify { + replace_rcpt chain { + step email_local_part + step email_with_domain example.org example.com + } +} +``` + +This configuration will alias `anything@anydomain` to `anything@example.org` +and `anything@example.com`. + +It is also useful with `authorize_sender` to authorize sending using multiple +addresses under different domains if non-email usernames are used for +authentication: + +``` +check.authorize_sender { + ... + user_to_email email_with_domain example.org example.com +} +``` + +This way, user authenticated as `user` will be allowed to use +`user@example.org` or `user@example.com` as a sender address. diff --git a/docs/reference/table/file.md b/docs/reference/table/file.md new file mode 100644 index 0000000..03254f0 --- /dev/null +++ b/docs/reference/table/file.md @@ -0,0 +1,58 @@ +# File + +table.file module builds string-string mapping from a text file. + +File is reloaded every 15 seconds if there are any changes (detected using +modification time). No changes are applied if file contains syntax errors. + +Definition: +``` +file +``` +or +``` +file { + file +} +``` + +Usage example: +``` +# Resolve SMTP address aliases using text file mapping. +modify { + replace_rcpt file /etc/maddy/aliases +} +``` + +## Syntax + +Better demonstrated by examples: + +``` +# Lines starting with # are ignored. + +# And so are lines only with whitespace. + +# Whenever 'aaa' is looked up, return 'bbb' +aaa: bbb + + # Trailing and leading whitespace is ignored. + ccc: ddd + +# If there is no colon, the string is translated into "" +# That is, the following line is equivalent to +# aaa: +aaa + +# If the same key is used multiple times - table.file will return +# multiple values when queries. +ddd: firstvalue +ddd: secondvalue + +# Alternatively, multiple values can be specified +# using a comma. There is no support for escaping +# so you would have to use a different format if you require +# comma-separated values. +ddd: firstvalue, secondvalue +``` + diff --git a/docs/reference/table/regexp.md b/docs/reference/table/regexp.md new file mode 100644 index 0000000..39e873d --- /dev/null +++ b/docs/reference/table/regexp.md @@ -0,0 +1,63 @@ +# Regexp rewrite table + +The 'regexp' module implements table lookups by applying a regular expression +to the key value. If it matches - 'replacement' value is returned with $N +placeholders being replaced with corresponding capture groups from the match. +Otherwise, no value is returned. + +The regular expression syntax is the subset of PCRE. See +[https://golang.org/pkg/regexp/syntax](https://golang.org/pkg/regexp/syntax)/ for details. + +``` +table.regexp [replacement] { + full_match yes + case_insensitive yes + expand_placeholders yes +} +``` + +Note that [replacement] is optional. If it is not included - table.regexp +will return the original string, therefore acting as a regexp match check. +This can be useful in combination in `destination_in` for +advanced matching: + +``` +destination_in regexp ".*-bounce+.*@example.com" { + ... +} +``` + +## Configuration directives + +### full_match _boolean_ +Default: `yes` + +Whether to implicitly add start/end anchors to the regular expression. +That is, if `full_match` is `yes`, then the provided regular expression should +match the whole string. With `no` - partial match is enough. + +--- + +### case_insensitive _boolean_ +Default: `yes` + +Whether to make matching case-insensitive. + +--- + +### expand_placeholders _boolean_ +Default: `yes` + +Replace '$name' and '${name}' in the replacement string with contents of +corresponding capture groups from the match. + +To insert a literal $ in the output, use $$ in the template. + +## Identity table (table.identity) + +The module 'identity' is a table module that just returns the key looked up. + +``` +table.identity { } +``` + diff --git a/docs/reference/table/sql_query.md b/docs/reference/table/sql_query.md new file mode 100644 index 0000000..9b3b9eb --- /dev/null +++ b/docs/reference/table/sql_query.md @@ -0,0 +1,120 @@ +# SQL query mapping + +The table.sql_query module implements table interface using SQL queries. + +Definition: + +``` +table.sql_query { + driver + dsn + lookup + + # Optional: + init + list + add + del + set +} +``` + +Usage example: + +``` +# Resolve SMTP address aliases using PostgreSQL DB. +modify { + replace_rcpt sql_query { + driver postgres + dsn "dbname=maddy user=maddy" + lookup "SELECT alias FROM aliases WHERE address = $1" + } +} +``` + +## Configuration directives + +### driver _driver name_ +**Required.** + +Driver to use to access the database. + +Supported drivers: `postgres`, `sqlite3` (if compiled with C support) + +--- + +### dsn _data source name_ +**Required.** + +Data Source Name to pass to the driver. For SQLite3 this is just a path to DB +file. For Postgres, see +[https://pkg.go.dev/github.com/lib/pq?tab=doc#hdr-Connection\_String\_Parameters](https://pkg.go.dev/github.com/lib/pq?tab=doc#hdr-Connection\_String\_Parameters) + +--- + +### lookup _query_ +**Required.** + +SQL query to use to obtain the lookup result. + +It will get one named argument containing the lookup key. Use :key +placeholder to access it in SQL. The result row set should contain one row, one +column with the string that will be used as a lookup result. If there are more +rows, they will be ignored. If there are more columns, lookup will fail. If +there are no rows, lookup returns "no results". If there are any error - lookup +will fail. + +--- + +### init _queries..._ +Default: empty + +List of queries to execute on initialization. Can be used to configure RDBMS. + +Example, to improve SQLite3 performance: + +``` +table.sql_query { + driver sqlite3 + dsn whatever.db + init "PRAGMA journal_mode=WAL" \ + "PRAGMA synchronous=NORMAL" + lookup "SELECT alias FROM aliases WHERE address = $1" +} +``` + +--- + +### named_args _boolean_ +Default: `yes` + +Whether to use named parameters binding when executing SQL queries +or not. + +Note that maddy's PostgreSQL driver does not support named parameters and +SQLite3 driver has issues handling numbered parameters: +[https://github.com/mattn/go-sqlite3/issues/472](https://github.com/mattn/go-sqlite3/issues/472) + +--- + +### add _query_
list _query_
set _query_
del _query_ +Default: none + +If queries are set to implement corresponding table operations - table becomes +"mutable" and can be used in contexts that require writable key-value store. + +'add' query gets :key, :value named arguments - key and value strings to store. +They should be added to the store. The query **should** not add multiple values +for the same key and **should** fail if the key already exists. + +'list' query gets no arguments and should return a column with all keys in +the store. + +'set' query gets :key, :value named arguments - key and value and should replace the existing +entry in the database. + +'del' query gets :key argument - key and should remove it from the database. + +If `named_args` is set to `no` - key is passed as the first numbered parameter +($1), value is passed as the second numbered parameter ($2). + diff --git a/docs/reference/table/static.md b/docs/reference/table/static.md new file mode 100644 index 0000000..e71b448 --- /dev/null +++ b/docs/reference/table/static.md @@ -0,0 +1,21 @@ +# Static table + +The 'static' module implements table lookups using key-value pairs in its +configuration. + +``` +table.static { + entry KEY1 VALUE1 + entry KEY2 VALUE2 + ... +} +``` + +## Configuration directives + +### entry _key_ _value_ + +Add an entry to the table. + +If the same key is used multiple times, the last one takes effect. + diff --git a/docs/reference/targets/queue.md b/docs/reference/targets/queue.md new file mode 100644 index 0000000..373ff1a --- /dev/null +++ b/docs/reference/targets/queue.md @@ -0,0 +1,95 @@ +# Local queue + +Queue module buffers messages on disk and retries delivery multiple times to +another target to ensure reliable delivery. + +It is also responsible for generation of DSN messages +in case of delivery failures. + +## Arguments + +First argument specifies directory to use for storage. +Relative paths are relative to the StateDirectory. + +## Configuration directives + +``` +target.queue { + target remote + location ... + max_parallelism 16 + max_tries 4 + bounce { + destination example.org { + deliver_to &local_mailboxes + } + default_destination { + reject + } + } + + autogenerated_msg_domain example.org + debug no +} +``` + +### target _block_name_ +**Required.**
+Default: not specified + +Delivery target to use for final delivery. + +--- + +### location _directory_ +Default: `StateDirectory/configuration_block_name` + +File system directory to use to store queued messages. +Relative paths are relative to the StateDirectory. + +--- + +### max_parallelism _integer_ +Default: `16` + +Start up to _integer_ goroutines for message processing. Basically, this option +limits amount of messages tried to be delivered concurrently. + +--- + +### max_tries _integer_ +Default: `20` + +Attempt delivery up to _integer_ times. Note that no more attempts will be done +is permanent error occurred during previous attempt. + +Delay before the next attempt will be increased exponentially using the +following formula: 15mins * 1.2 ^ (n - 1) where n is the attempt number. +This gives you approximately the following sequence of delays: +18mins, 21mins, 25mins, 31mins, 37mins, 44mins, 53mins, 64mins, ... + +--- + +### bounce { ... } +Default: not specified + +This configuration contains pipeline configuration to be used for generated DSN +(Delivery Status Notification) messages. + +If this is block is not present in configuration, DSNs will not be generated. +Note, however, this is not what you want most of the time. + +--- + +### autogenerated_msg_domain _domain_ +Default: global directive value + +Domain to use in sender address for DSNs. Should be specified too if 'bounce' +block is specified. + +--- + +### debug _boolean_ +Default: `no` + +Enable verbose logging. \ No newline at end of file diff --git a/docs/reference/targets/remote.md b/docs/reference/targets/remote.md new file mode 100644 index 0000000..9a1b606 --- /dev/null +++ b/docs/reference/targets/remote.md @@ -0,0 +1,295 @@ +# Remote MX delivery + +Module that implements message delivery to remote MTAs discovered via DNS MX +records. You probably want to use it with queue module for reliability. + +If a message check marks a message as 'quarantined', remote module +will refuse to deliver it. + +## Configuration directives + +``` +target.remote { + hostname mx.example.org + debug no +} +``` + +### hostname _domain_ +Default: global directive value + +Hostname to use client greeting (EHLO/HELO command). Some servers require it to +be FQDN, SPF-capable servers check whether it corresponds to the server IP +address, so it is better to set it to a domain that resolves to the server IP. + +--- + +### limits { ... } +Default: no limits + +See ['limits' directive for SMTP endpoint](/reference/endpoints/smtp/#rate-concurrency-limiting). +It works the same except for address domains used for +per-source/per-destination are as observed when message exits the server. + +--- + +### local_ip _ip-address_ +Default: empty + +Choose the local IP to bind for outbound SMTP connections. + +--- + +### force_ipv4 _boolean_ +Default: `false` + +Force resolving outbound SMTP domains to IPv4 addresses. Some server providers +do not offer a way to properly set reverse PTR domains for IPv6 addresses; this +option makes maddy only connect to IPv4 addresses so that its public IPv4 address +is used to connect to that server, and thus reverse PTR checks are made against +its IPv4 address. + +Warning: this may break sending outgoing mail to IPv6-only SMTP servers. + +--- + +### connect_timeout _duration_ +Default: `5m` + +Timeout for TCP connection establishment. + +RFC 5321 recommends 5 minutes for "initial greeting" that includes TCP +handshake. maddy uses two separate timers - one for "dialing" (DNS A/AAAA +lookup + TCP handshake) and another for "initial greeting". This directive +configures the former. The latter is not configurable and is hardcoded to be +5 minutes. + +--- + +### command_timeout _duration_ +Default: `5m` + +Timeout for any SMTP command (EHLO, MAIL, RCPT, DATA, etc). + +If STARTTLS is used this timeout also applies to TLS handshake. + +RFC 5321 recommends 5 minutes for MAIL/RCPT and 3 minutes for +DATA. + +--- + +### submission_timeout _duration_ +Default: `12m` + +Time to wait after the entire message is sent (after "final dot"). + +RFC 5321 recommends 10 minutes. + +--- + +### debug _boolean_ +Default: global directive value + +Enable verbose logging. + +--- + +### requiretls_override _boolean_ +Default: `true` + +Allow local security policy to be disabled using 'TLS-Required' header field in +sent messages. Note that the field has no effect if transparent forwarding is +used, message body should be processed before outbound delivery starts for it +to take effect (e.g. message should be queued using 'queue' module). + +--- + +### relaxed_requiretls _boolean_ +Default: `true` + +This option disables strict conformance with REQUIRETLS specification and +allows forwarding of messages 'tagged' with REQUIRETLS to MXes that are not +advertising REQUIRETLS support. It is meant to allow REQUIRETLS use without the +need to have support from all servers. It is based on the assumption that +server referenced by MX record is likely the final destination and therefore +there is only need to secure communication towards it and not beyond. + +--- + +### conn_reuse_limit _integer_ +Default: `10` + +Amount of times the same SMTP connection can be used. +Connections are never reused if the previous DATA command failed. + +--- + +### conn_max_idle_count _integer_ +Default: `10` + +Max. amount of idle connections per recipient domains to keep in cache. + +--- + +### conn_max_idle_time _integer_ +Default: `150` (2.5 min) + +Amount of time the idle connection is still considered potentially usable. + +--- + +## Security policies + +### mx_auth { ... } +Default: no policies + +'remote' module implements a number of of schemes and protocols necessary to +ensure security of message delivery. Most of these schemes are concerned with +authentication of recipient server and TLS enforcement. + +To enable mechanism, specify its name in the `mx_auth` directive block: + +``` +mx_auth { + dane + mtasts +} +``` + +Additional configuration is possible if supported by the mechanism by +specifying additional options as a block for the corresponding mechanism. +E.g. + +``` +mtasts { + cache ram +} +``` + +If the `mx_auth` directive is not specified, no mechanisms are enabled. Note +that, however, this makes outbound SMTP vulnerable to a numerous downgrade +attacks and hence not recommended. + +It is possible to share the same set of policies for multiple 'remote' module +instances by defining it at the top-level using `mx_auth` module and then +referencing it using standard & syntax: + +``` +mx_auth outbound_policy { + dane + mtasts { + cache ram + } +} + +# ... somewhere else ... + +deliver_to remote { + mx_auth &outbound_policy +} + +# ... somewhere else ... + +deliver_to remote { + mx_auth &outbound_policy + tls_client { ... } +} +``` + +--- + +### MTA-STS + +Checks MTA-STS policy of the recipient domain. Provides proper authentication +and TLS enforcement for delivery, but partially vulnerable to persistent active +attacks. + +Sets MX level to "mtasts" if the used MX matches MTA-STS policy even if it is +not set to "enforce" mode. + +``` +mtasts { + cache fs + fs_dir StateDirectory/mtasts_cache +} +``` + +### cache `fs` | `ram` +Default: `fs` + +Storage to use for MTA-STS cache. 'fs' is to use a filesystem directory, 'ram' +to store the cache in memory. + +It is recommended to use 'fs' since that will not discard the cache (and thus +cause MTA-STS security to disappear) on server restart. However, using the RAM +cache can make sense for high-load configurations with good uptime. + +### fs_dir _directory_ +Default: `StateDirectory/mtasts_cache` + +Filesystem directory to use for policies caching if 'cache' is set to 'fs'. + +--- + +### DNSSEC + +Checks whether MX records are signed. Sets MX level to "dnssec" is they are. + +maddy does not validate DNSSEC signatures on its own. Instead it relies on +the upstream resolver to do so by causing lookup to fail when verification +fails and setting the AD flag for signed and verified zones. As a safety +measure, if the resolver is not 127.0.0.1 or ::1, the AD flag is ignored. + +DNSSEC is currently not supported on Windows and other platforms that do not +have the /etc/resolv.conf file in the standard format. + +``` +dnssec { } +``` + +--- + +### DANE + +Checks TLSA records for the recipient MX. Provides downgrade-resistant TLS +enforcement. + +Sets TLS level to "authenticated" if a valid and matching TLSA record uses +DANE-EE or DANE-TA usage type. + +See above for notes on DNSSEC. DNSSEC support is required for DANE to work. + +``` +dane { } +``` + +--- + +### Local policy + +Checks effective TLS and MX levels (as set by other policies) against local +configuration. + +``` +local_policy { + min_tls_level none + min_mx_level none +} +``` + +Using `local_policy off` is equivalent to setting both directives to `none`. + +### min_tls_level `none` | `encrypted` | `authenticated` +Default: `encrypted` + +Set the minimal TLS security level required for all outbound messages. + +See [Security levels](/seclevels) page for details. + +### min_mx_level `none` | `mtasts` | `dnssec` +Default: `none` + +Set the minimal MX security level required for all outbound messages. + +See [Security levels](/seclevels) page for details. + diff --git a/docs/reference/targets/smtp.md b/docs/reference/targets/smtp.md new file mode 100644 index 0000000..ebbc430 --- /dev/null +++ b/docs/reference/targets/smtp.md @@ -0,0 +1,123 @@ +# SMTP & LMTP transparent forwarding + +Module that implements transparent forwarding of messages over SMTP. + +Use in pipeline configuration: + +``` +deliver_to smtp tcp://127.0.0.1:5353 +# or +deliver_to smtp tcp://127.0.0.1:5353 { + # Other settings, see below. +} +``` + +target.lmtp can be used instead of target.smtp to +use LMTP protocol. + +Endpoint addresses use format described in [Configuration files syntax / Address definitions](/reference/config-syntax/#address-definitions). + +## Configuration directives + +``` +target.smtp { + debug no + tls_client { + ... + } + attempt_starttls yes + require_tls no + auth off + targets tcp://127.0.0.1:2525 + connect_timeout 5m + command_timeout 5m + submission_timeout 12m +} +``` + +### debug _boolean_ +Default: global directive value + +Enable verbose logging. + +--- + +### tls_client { ... } +Default: not specified + +Advanced TLS client configuration options. See [TLS configuration / Client](/reference/tls/#client) for details. + +--- + +### starttls _boolean_ +Default: `yes` (`no` for `target.lmtp`) + +Use STARTTLS to enable TLS encryption. If STARTTLS is not supported +by the remote server - connection will fail. + +maddy will use `localhost` as HELO hostname before STARTTLS +and will only send its actual hostname after STARTTLS. + +### attempt_starttls _boolean_ +Default: `yes` (`no` for `target.lmtp`) + +DEPRECATED: Equivalent to `starttls`. Plaintext fallback is no longer +supported. + +--- + +### require_tls _boolean_ +Default: `no` + +DEPRECATED: Ignored. Set `starttls yes` to use STARTLS. + +--- + +### auth `off` | `plain` _username_ _password_ | `forward` | `external` +Default: `off` + +Specify the way to authenticate to the remote server. +Valid values: + +- `off` – No authentication. +- `plain` – Authenticate using specified username-password pair. + **Don't use** this without enforced TLS (`require_tls`). +- `forward` – Forward credentials specified by the client. + **Don't use** this without enforced TLS (`require_tls`). +- `external` – Request "external" SASL authentication. This is usually used for + authentication using TLS client certificates. See [TLS configuration / Client](/reference/tls/#client) for details. + +--- + +### targets _endpoints..._ +**Required.**
+Default: not specified + +List of remote server addresses to use. See [Address definitions](/reference/config-syntax/#address-definitions) +for syntax to use. Basically, it is `tcp://ADDRESS:PORT` +for plain SMTP and `tls://ADDRESS:PORT` for SMTPS (aka SMTP with Implicit +TLS). + +Multiple addresses can be specified, they will be tried in order until connection to +one succeeds (including TLS handshake if TLS is required). + +--- + +### connect_timeout _duration_ +Default: `5m` + +Same as for target.remote. + +--- + +### command_timeout _duration_ +Default: `5m` + +Same as for target.remote. + +--- + +### submission_timeout _duration_ +Default: `12m` + +Same as for target.remote. diff --git a/docs/reference/tls-acme.md b/docs/reference/tls-acme.md new file mode 100644 index 0000000..a7be47d --- /dev/null +++ b/docs/reference/tls-acme.md @@ -0,0 +1,290 @@ +# Automatic certificate management via ACME + +Maddy supports obtaining certificates using ACME protocol. + +To use it, create a configuration name for `tls.loader.acme` +and reference it from endpoints that should use automatically +configured certificates: + +``` +tls.loader.acme local_tls { + email put-your-email-here@example.org + agreed # indicate your agreement with Let's Encrypt ToS + challenge dns-01 +} + +smtp tcp://127.0.0.1:25 { + tls &local_tls + ... +} +``` + +You can also use a global `tls` directive to use automatically +obtained certificates for all endpoints: + +``` +tls { + loader acme { + email maddy-acme@example.org + agreed + challenge dns-01 + } +} +``` + +Note: `tls &local_tls` as a global directive won't work because +global directives are initialized before other configuration blocks. + +Currently the only supported challenge is `dns-01` one therefore +you also need to configure the DNS provider: + +``` +tls.loader.acme local_tls { + email maddy-acme@example.org + agreed + challenge dns-01 + dns PROVIDER_NAME { + ... + } +} +``` + +See below for supported providers and necessary configuration +for each. + +## Configuration directives + +``` +tls.loader.acme { + debug off + hostname example.maddy.invalid + store_path /var/lib/maddy/acme + ca https://acme-v02.api.letsencrypt.org/directory + test_ca https://acme-staging-v02.api.letsencrypt.org/directory + email test@maddy.invalid + agreed off + challenge dns-01 + dns ... +} +``` + +### debug _boolean_ +Default: global directive value + +Enable debug logging. + +--- + +### hostname _str_ +**Required.**
+Default: global directive value + +Domain name to issue certificate for. + +--- + +### store_path _path_ +Default: `state_dir/acme` + +Where to store issued certificates and associated metadata. +Currently only filesystem-based store is supported. + +--- + +### ca _url_ +Default: Let's Encrypt production CA + +URL of ACME directory to use. + +--- + +### test_ca _url_ +Default: Let's Encrypt staging CA + +URL of ACME directory to use for retries should +primary CA fail. + +maddy will keep attempting to issues certificates +using `test_ca` until it succeeds then it will switch +back to the one configured via 'ca' option. + +This avoids rate limit issues with production CA. + +--- + +### override_domain _domain_ +Default: not set + +Override the domain to set the TXT record on for DNS-01 challenge. +This is to delegate the challenge to a different domain. + +See https://www.eff.org/deeplinks/2018/02/technical-deep-dive-securing-automation-acme-dns-challenge-validation +for explanation why this might be useful. + +--- + +### email _str_ +Default: not set + +Email to pass while registering an ACME account. + +--- + +### agreed _boolean_ +Default: false + +Whether you agreed to ToS of the CA service you are using. + +--- + +### challenge `dns-01` +Default: not set + +Challenge(s) to use while performing domain verification. + +## DNS providers + +Support for some providers is not provided by standard builds. +To be able to use these, you need to compile maddy +with "libdns_PROVIDER" build tag. +E.g. +``` +./build.sh --tags 'libdns_googleclouddns' +``` + +- gandi + +``` +dns gandi { + api_token "token" +} +``` + +- digitalocean + +``` +dns digitalocean { + api_token "..." +} +``` + +- cloudflare + +See [https://github.com/libdns/cloudflare#authenticating](https://github.com/libdns/cloudflare#authenticating) + +``` +dns cloudflare { + api_token "..." +} +``` + +- vultr + +``` +dns vultr { + api_token "..." +} +``` + +- hetzner + +``` +dns hetzner { + api_token "..." +} +``` + +- namecheap + +``` +dns namecheap { + api_key "..." + api_username "..." + + # optional: API endpoint, production one is used if not set. + endpoint "https://api.namecheap.com/xml.response" + + # optional: your public IP, discovered using icanhazip.com if not set + client_ip 1.2.3.4 +} +``` + +- googleclouddns (non-default) + +``` +dns googleclouddns { + project "project_id" + service_account_json "path" +} +``` + +- route53 (non-default) + +``` +dns route53 { + secret_access_key "..." + access_key_id "..." + # or use environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +} +``` + +- leaseweb (non-default) + +``` +dns leaseweb { + api_key "key" +} +``` + +- metaname (non-default) + +``` +dns metaname { + api_key "key" + account_ref "reference" +} +``` + +- alidns (non-default) + +``` +dns alidns { + key_id "..." + key_secret "..." +} +``` + +- namedotcom (non-default) + +``` +dns namedotcom { + user "..." + token "..." +} +``` + +- rfc2136 (non-default) + +``` +dns rfc2136 { + key_name "..." + # Secret + key "..." + # HMAC algorithm used to generate the key, lowercase, e.g. hmac-sha512 + key_alg "..." + # server to which the dynamic update will be sent, e.g. 127.0.0.1 + # you can also specify the port: 127.0.0.1:53 + server "..." +} +``` + +- acmedns (non-default) + +``` +dns acmedns { + username "..." + password "..." + subdomain "..." + server_url "..." +} +``` diff --git a/docs/reference/tls.md b/docs/reference/tls.md new file mode 100644 index 0000000..954b0e0 --- /dev/null +++ b/docs/reference/tls.md @@ -0,0 +1,155 @@ +# TLS configuration + +## Server-side + +TLS certificates are obtained by modules called "certificate loaders". 'tls' directive +arguments specify name of loader to use and arguments. Due to syntax limitations +advanced configuration for loader should be specified using 'loader' directive, see +below. + +``` +tls file cert.pem key.pem { + protocols tls1.2 tls1.3 + curves X25519 + ciphers ... +} + +tls { + loader file cert.pem key.pem { + # Options for loader go here. + } + protocols tls1.2 tls1.3 + curves X25519 + ciphers ... +} +``` + +### Available certificate loaders + +- `file` – Accepts argument pairs specifying certificate and then key. + E.g. `tls file certA.pem keyA.pem certB.pem keyB.pem`. + If multiple certificates are listed, SNI will be used. +- `acme` – Automatically obtains a certificate using ACME protocol (Let's Encrypt) +- `off` – Not really a loader but a special value for tls directive, + explicitly disables TLS for endpoint(s). + +## Advanced TLS configuration + +**Note: maddy uses secure defaults and TLS handshake is resistant to active downgrade attacks. There is no need to change anything in most cases.** + +--- + +### protocols _min-version_ _max-version_ | _version_ +Default: `tls1.0 tls1.3` + +Minimum/maximum accepted TLS version. If only one value is specified, it will +be the only one usable version. + +Valid values are: `tls1.0`, `tls1.1`, `tls1.2`, `tls1.3` + +--- + +### ciphers _ciphers..._ +Default: Go version-defined set of 'secure ciphers', ordered by hardware +performance + +List of supported cipher suites, in preference order. Not used with TLS 1.3. + +Valid values: + +- `RSA-WITH-RC4128-SHA` +- `RSA-WITH-3DES-EDE-CBC-SHA` +- `RSA-WITH-AES128-CBC-SHA` +- `RSA-WITH-AES256-CBC-SHA` +- `RSA-WITH-AES128-CBC-SHA256` +- `RSA-WITH-AES128-GCM-SHA256` +- `RSA-WITH-AES256-GCM-SHA384` +- `ECDHE-ECDSA-WITH-RC4128-SHA` +- `ECDHE-ECDSA-WITH-AES128-CBC-SHA` +- `ECDHE-ECDSA-WITH-AES256-CBC-SHA` +- `ECDHE-RSA-WITH-RC4128-SHA` +- `ECDHE-RSA-WITH-3DES-EDE-CBC-SHA` +- `ECDHE-RSA-WITH-AES128-CBC-SHA` +- `ECDHE-RSA-WITH-AES256-CBC-SHA` +- `ECDHE-ECDSA-WITH-AES128-CBC-SHA256` +- `ECDHE-RSA-WITH-AES128-CBC-SHA256` +- `ECDHE-RSA-WITH-AES128-GCM-SHA256` +- `ECDHE-ECDSA-WITH-AES128-GCM-SHA256` +- `ECDHE-RSA-WITH-AES256-GCM-SHA384` +- `ECDHE-ECDSA-WITH-AES256-GCM-SHA384` +- `ECDHE-RSA-WITH-CHACHA20-POLY1305` +- `ECDHE-ECDSA-WITH-CHACHA20-POLY1305` + +--- + +### curves _curves..._ +Default: defined by Go version + +The elliptic curves that will be used in an ECDHE handshake, in preference +order. + +Valid values: `p256`, `p384`, `p521`, `X25519`. + +## Client + +`tls_client` directive allows to customize behavior of TLS client implementation, +notably adjusting minimal and maximal TLS versions and allowed cipher suites, +enabling TLS client authentication. + +``` +tls_client { + protocols tls1.2 tls1.3 + ciphers ... + curves X25519 + root_ca /etc/ssl/cert.pem + + cert /etc/ssl/private/maddy-client.pem + key /etc/ssl/private/maddy-client.pem +} +``` + +--- + +### protocols _min-version_ _max-version_ | _version_ +Default: `tls1.0 tls1.3` + +Minimum/maximum accepted TLS version. If only one value is specified, it will +be the only one usable version. + +Valid values are: `tls1.0`, `tls1.1`, `tls1.2`, `tls1.3` + +--- + +### ciphers _ciphers..._ +Default: Go version-defined set of 'secure ciphers', ordered by hardware +performance + +List of supported cipher suites, in preference order. Not used with TLS 1.3. + +See TLS server configuration for list of supported values. + +--- + +### curves _curves..._ +Default: defined by Go version + +The elliptic curves that will be used in an ECDHE handshake, in preference +order. + +Valid values: `p256`, `p384`, `p521`, `X25519`. + +--- + +### root_ca _paths..._ +Default: system CA pool + +List of files with PEM-encoded CA certificates to use when verifying +server certificates. + +--- + +### cert _cert-path_
key _key-path_ +Default: not specified + +Present the specified certificate when server requests a client certificate. +Files should use PEM format. Both directives should be specified. diff --git a/docs/seclevels.md b/docs/seclevels.md new file mode 100644 index 0000000..984be4a --- /dev/null +++ b/docs/seclevels.md @@ -0,0 +1,89 @@ +# Outbound delivery security + +maddy implements a number of schemes and protocols for discovery and +enforcement of security features supported by the recipient MTA. + +## Introduction to the problems of secure SMTP + +Outbound delivery security involves two independent problems: + +- MX record authentication +- TLS enforcement + +### MX record authentication + +When MTA wants to deliver a message to a mailbox at remote domain, it needs to +discover the server to use for it. It is done through the lookup of DNS MX +records for the recipient. + +Problem arises from the fact that DNS does not have any cryptographic +protection and so any malicious actor can technically modify the response to +contain any server. And MTA would use that server! + +There are two protocols that solve this problem: MTA-STS and DNSSEC. +Former requires the MTA to verify used records against a list of rules published +via HTTPS. Later cryptographically signs the records themselves. + +### TLS enforcement + +By default, server-server SMTP is unencrypted. If remote server supports TLS, +it is advertised via the ESMTP extension named STARTTLS, but malicious actor +controlling communication channel can hide the support for STARTTLS and sender +MTA will have to use plaintext. There needs to be a out-of-band authenticated +channel to indicate TLS support (and to require its use). + +MTA-STS and DANE solve this problem. In the first case, if policy is in +"enforce" mode then MTA is required to use TLS when delivering messages to a +remote server. DANE does pretty much the same thing, but using DNSSEC-signed +TLSA records. + +## maddy policy details + +maddy defines two values indicating how "secure" delivery of message will be: + +- MX security level +- TLS security level + +These values correspond to the problems described above. On delivery, the +established connection to the remote server is "ranked" using these values and +then they are compared against a number of policies (including local +configuration). If the effective value is lower than the required one, the +connection is closed and next candidate server is used. If all connections fail +this way - the delivery is failed (or deferred if there was a temporary error +when checking policies). + +Below is the table summarizing the security level values defined in maddy and +protection they offer. + +| MX/TLS level | None | Encrypted | Authenticated | +| ------------- | ---- | --------- | -------------------- | +| None | - | P | P | +| MTA-STS | - | P | PA (see note 1) | +| DNSSEC | - | P | PA | + +Legend: P - protects against passive attacks; A - protects against active +attacks + +- MX level: None. MX candidate was returned as a result of DNS lookup for the + recipient domain, no additional checks done. +- MX level: MTA-STS. Used MX matches the MTA-STS policy published by the + recipient domain (even one in testing mode). +- MX level: DNSSEC. MX record is signed. + +- TLS level: None. Plaintext connection was established, TLS is not available + or failed. +- TLS level: Encrypted. TLS connection was established, the server certificate + failed X.509 and DANE verification. +- TLS level: Authenticated. TLS connection was established, the server + certificate passes X.509 **or** DANE verification. + +**Note 1:** Persistent attacker able to control network connection can +interfere with policy refresh, downgrading protection to be secure only against +passive attacks. + +## maddy security policies + +See [Remote MX delivery](/reference/targets/remote/) for description of configuration options available for each policy mechanism +supported by maddy. + +[RFC 8461 Section 10.2]: https://www.rfc-editor.org/rfc/rfc8461.html#section-10.2 (SMTP MTA Strict Transport Security - 10.2. Preventing Policy Discovery) diff --git a/docs/third-party/dovecot.md b/docs/third-party/dovecot.md new file mode 100644 index 0000000..22d51c3 --- /dev/null +++ b/docs/third-party/dovecot.md @@ -0,0 +1,87 @@ +# Dovecot + +Builtin maddy IMAP server may not match your requirements in terms of +performance, reliability or anything. For this reason it is possible to +integrate it with any external IMAP server that implements necessary +protocols. Here is how to do it for Dovecot. + +1. Get rid of `imap` endpoint and existing `local_authdb` and `local_mailboxes` + blocks. + +2. Setup Dovecot to provide LMTP endpoint + +Here is an example configuration snippet: +``` +# /etc/dovecot/dovecot.conf +protocols = imap lmtp + +# /etc/dovecot/conf.d/10-master.conf +service lmtp { + unix_listener lmtp-maddy { + mode = 0600 + user = maddy + } +} +``` + +Add `local_mailboxes` block to maddy config using `target.lmtp` module: +``` +target.lmtp local_mailboxes { + targets unix:///var/run/dovecot/lmtp-maddy +} +``` + +### Authentication + +In addition to MTA service, maddy also provides Submission service, but it +needs authentication provider data to work correctly, maddy can use Dovecot +SASL authentication protocol for it. + +You need the following in Dovecot's `10-master.conf`: +``` +service auth { + unix_listener auth-maddy-client { + mode = 0660 + user = maddy + } +} +``` + +Then just configure `dovecot_sasl` module for `submission`: +``` +submission ... { + auth dovecot_sasl unix:///var/run/dovecot/auth-maddy-client + ... other configuration ... +} +``` + +## Other IMAP servers + +Integration with other IMAP servers might be more problematic because there is +no standard protocol for authentication delegation. You might need to configure +the IMAP server to implement MSA functionality by forwarding messages to maddy +for outbound delivery. This might require more configuration changes on maddy +side since by default it will not allow relay on port 25 even for localhost +addresses. The easiest way is to create another SMTP endpoint on some port +(probably Submission port): +``` +smtp tcp://127.0.0.1:587 { + deliver_to &remote_queue +} +``` +And configure IMAP server's Submission service to forward outbound messages +there. + +Depending on how Submission service is implemented you may also need to route +messages for local domains back to it via LMTP: +``` +smtp tcp://127.0.0.1:587 { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + deliver_to &remote_queue + } +} +``` + diff --git a/docs/third-party/mailman3.md b/docs/third-party/mailman3.md new file mode 100644 index 0000000..a29d71e --- /dev/null +++ b/docs/third-party/mailman3.md @@ -0,0 +1,84 @@ +# Mailman 3 + +Setting up Mailman 3 with maddy involves some additional work as compared to +other MTAs as there is no Python package in Mailman suite that can generate +address lists in format supported by maddy. + +We assume you are already familiar with Mailman configuration guidelines and +how stuff works in general/for other MTAs. + +## Accepting messages + +First of all, you need to use NullMTA package for mta.incoming so Mailman will +not try to generate any configs. LMTP listener is configured as usual. +``` +[mta] +incoming: mailman.mta.null.NullMTA +lmtp_host: 127.0.0.1 +lmtp_port: 8024 +``` + +After that, you will need to configure maddy to send messages to Mailman. + +The preferable way of doing so is destination_in and table.regexp: +``` +msgpipeline local_routing { + destination_in regexp "first-mailinglist(-(bounces\+.*|confirm\+.*|join|leave|owner|request|subscribe|unsubscribe))?@lists.example.org" { + deliver_to lmtp tcp://127.0.0.1:8024 + } + destination_in regexp "second-mailinglist(-(bounces\+.*|confirm\+.*|join|leave|owner|request|subscribe|unsubscribe))?@lists.example.org" { + deliver_to lmtp tcp://127.0.0.1:8024 + } + + ... +} +``` + +A more simple option is also meaningful (provided you have a separate domain +for lists): +``` +msgpipeline local_routing { + destination lists.example.org { + deliver_to lmtp tcp://127.0.0.1:8024 + } + + ... +} +``` +But this variant will lead to inefficient handling of non-existing subaddresses. +See [Mailman Core issue 14](https://gitlab.com/mailman/mailman/-/issues/14) for +details. (5 year old issue, sigh...) + +## Sending messages + +It is recommended to configure Mailman to send messages using Submission port +with authentication and TLS as maddy does not allow relay on port 25 for local +clients as some MTAs do: +``` +[mta] +# ... incoming configuration here ... +outgoing: mailman.mta.deliver.deliver +smtp_host: mx.example.org +smtp_port: 465 +smtp_user: mailman@example.org +smtp_pass: something-very-secret +smtp_secure_mode: smtps +``` + +If you do not want to use TLS and/or authentication you can create a separate +endpoint and just point Mailman to it. E.g. +``` +smtp tcp://127.0.0.1:2525 { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + deliver_to &remote_queue + } +} +``` + +Note that if you use a separate domain for lists, it need to be included in +local_domains macro in default config. This will ensure maddy signs messages +using DKIM for outbound messages. It is also highly recommended to configure +ARC in Mailman 3. diff --git a/docs/third-party/rspamd.md b/docs/third-party/rspamd.md new file mode 100644 index 0000000..3d2ce48 --- /dev/null +++ b/docs/third-party/rspamd.md @@ -0,0 +1,38 @@ +# rspamd + +maddy has direct support for rspamd HTTP protocol. There is no need to use +milter proxy. + +If rspamd is running locally, it is enough to just add `rspamd` check +with default configuration into appropriate check block (probably in +local_routing): +``` +check { + ... + rspamd +} +``` + +You might want to disable builtin SPF, DKIM and DMARC for performance +reasons but note that at the moment, maddy will not generate +Authentication-Results field with rspamd results. + +If rspamd is not running on a local machine, change api_path to point +to the "normal" worker socket: + +``` +check { + ... + rspamd { + api_path http://spam-check.example.org:11333 + } +} +``` + +Default mapping of rspamd action -> maddy action is as follows: + +- "add header" => Quarantine +- "rewrite subject" => Quarantine +- "soft reject" => Reject with temporary error +- "reject" => Reject with permanent error +- "greylist" => Ignored \ No newline at end of file diff --git a/docs/third-party/smtp-servers.md b/docs/third-party/smtp-servers.md new file mode 100644 index 0000000..599a00d --- /dev/null +++ b/docs/third-party/smtp-servers.md @@ -0,0 +1,58 @@ +# External SMTP server + +It is possible to use maddy as an IMAP server only and have it interface with +external SMTP server using standard protocols. + +Here is the minimal configuration that creates a local IMAP index, credentials +database and IMAP endpoint: +``` +# Credentials DB. +table.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } +} + +# IMAP storage/index. +storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db +} + +# IMAP endpoint using these above. +imap tls://0.0.0.0:993 tcp://0.0.0.0:143 { + auth &local_authdb + storage &local_mailboxes +} +``` + +To accept local messages from an external SMTP server +it is possible to create an LMTP endpoint: +``` +# LMTP endpoint on Unix socket delivering to IMAP storage +# in previous config snippet. +lmtp unix:/run/maddy/lmtp.sock { + hostname mx.maddy.test + + deliver_to &local_mailboxes +} +``` + +Look up documentation for your SMTP server on how to make it +send messages using LMTP to /run/maddy/lmtp.sock. + +To handle authentication for Submission (client-server SMTP) SMTP server +needs to access credentials database used by maddy. maddy implements +server side of Dovecot authentication protocol so you can use +it if SMTP server implements "Dovecot SASL" client. + +To create a Dovecot-compatible sasld endpoint, add the following configuration +block: +``` +# Dovecot-compatible sasld endpoint using data from local_authdb. +dovecot_sasld unix:/run/maddy/auth-client.sock { + auth &local_authdb +} +``` diff --git a/docs/tutorials/alias-to-remote.md b/docs/tutorials/alias-to-remote.md new file mode 100644 index 0000000..ddbc76b --- /dev/null +++ b/docs/tutorials/alias-to-remote.md @@ -0,0 +1,123 @@ +# Forward messages to a remote MX + +Default maddy configuration is done in a way that does not result in any +outbound messages being sent as a result of port 25 traffic. + +In particular, this means that if you handle messages for example.org but not +example.com and have the following in your aliases file (e.g. /etc/maddy/aliases): + +``` +foxcpp@example.org: foxcpp@example.com +``` + +You will get "User does not exist" error when attempting to send a message to +foxcpp@example.org because foxcpp@example.com does not exist on as a local +user. + +Some users may want to make it work, but it is important to understand the +consequences of such configuration: + +- Flooding your server will also flood the remote server. +- If your spam filtering is not good enough, you will send spam to the remote + server. + +In both cases, you might harm the reputation of your server (e.g. get your IP +listed in a DNSBL). + +**So, this is a bad practice. Do so only if you clearly understand the +consequences (including the Bounce handling section below).** + +If you want to do it anyway, here is the part of the configuration that needs +tweaking: + +``` +msgpipeline local_routing { + destination postmaster $(local_domains) { + modify { + replace_rcpt regexp "(.+)\+(.+)@(.+)" "$1@$3" + replace_rcpt file /etc/maddy/aliases + } + + deliver_to &local_mailboxes + } + + default_destination { + reject 550 5.1.1 "User doesn't exist" + } +} +``` + +In default configuration, `local_routing` block is responsible for handling +messages that are received via SMTP or Submission and have the initial +destination address at a local domain. + +Note the `modify { }` block being nested inside `destination` and then followed +by unconditional `deliver_to &local_mailboxes`. This means: if address is +on `$(local_domains)`, apply aliases and deliver to mailboxes from +`&local_mailboxes`. + +The problem here is that recipients are matched before aliases are resolved so +in the end, maddy attempts to look up foxcpp@example.com locally. The solution +is to insert another step into the pipeline configuration to rerun matching +*after* aliases are resolved. This can be done using the 'reroute' directive: + +``` +msgpipeline local_routing { + destination postmaster $(local_domains) { + modify { + replace_rcpt file /etc/maddy/aliases + ... + } + + reroute { + destination postmaster $(local_domains) { + deliver_to &local_mailboxes + } + default_destination { + deliver_to &remote_queue + } + } + } + + default_destination { + reject 550 5.1.1 "User doesn't exist" + } +} +``` + +## Bounce handling + +Once the message is delivered to `remote_queue`, it will follow the usual path +for outbound delivery, including queuing and multiple attempts. This also +means bounce messages will be generated on failures. When accepting messages +from arbitrary senders via the 25 port, the DSN recipient will be whatever +sender specifies in the MAIL FROM command. This is prone to [collateral spam] +when an automatically generated bounce message gets sent to a spoofed address. + +However, the default maddy configuration ensures that in this case, the NDN +will be delivered only if the original sender is a local user. Backscatter can +not happen if the sender spoofed a local address since such messages will not +be accepted in the first place. + +You can also configure maddy to send bounce messages to remote +addresses, but in this case, you should configure a really strict local policy +to make sure the sender address is not spoofed. There is no detailed +explanation of how to do this since this is a terrible idea in general. + +[collateral spam]: https://en.wikipedia.org/wiki/Backscatter_(e-mail) + +## Transparent forwarding + +As an alternative to silently dropping messages on remote delivery failures, +you might want to use transparent forwarding and reject the message without +accepting it first ("connection-stage rejection"). + +To do so, simply do not use the queue, replace +``` +deliver_to &remote_queue +``` +with +``` +deliver_to &outbound_delivery +``` +(assuming outbound_delivery refers to target.remote block) diff --git a/docs/tutorials/building-from-source.md b/docs/tutorials/building-from-source.md new file mode 100644 index 0000000..55b6852 --- /dev/null +++ b/docs/tutorials/building-from-source.md @@ -0,0 +1,55 @@ +# Building from source + +## System dependencies + +You need C toolchain, Go toolchain and Make: + +On Debian-based system this should work: +``` +apt-get install golang-1.23 gcc libc6-dev make +``` + +Additionally, if you want manual pages, you should also have scdoc installed. +Figuring out the appropriate way to get scdoc is left as an exercise for +reader (for Ubuntu 22.04 LTS it is in repositories). + +## Recent Go toolchain + +maddy depends on a rather recent Go toolchain version that may not be +available in some distributions (*cough* Debian *cough*). + +`go` command in Go 1.21 or newer will automatically download up-to-date +toolchain to build maddy. It is necessary to run commands below only +if you have `go` command version older than 1.21. + +``` +wget "https://go.dev/dl/go1.23.5.linux-amd64.tar.gz" +tar xf "go1.23.5.linux-amd64.tar.gz" +export GOROOT="$PWD/go" +export PATH="$PWD/go/bin:$PATH" +``` + +## Step-by-step + +1. Clone repository +``` +$ git clone https://github.com/foxcpp/maddy.git +$ cd maddy +``` + +2. Select the appropriate version to build: +``` +$ git checkout v0.8.0 # a specific release +$ git checkout master # next bugfix release +$ git checkout dev # next feature release +``` + +3. Build & install it +``` +$ ./build.sh +$ sudo ./build.sh install +``` + +4. Finish setup as described in [Setting up](../setting-up) (starting from System configuration). + + diff --git a/docs/tutorials/pam.md b/docs/tutorials/pam.md new file mode 100644 index 0000000..a189a33 --- /dev/null +++ b/docs/tutorials/pam.md @@ -0,0 +1,94 @@ +# Using PAM authentication + +maddy supports user authentication using PAM infrastructure via `auth.pam` +module. + +In order to use it, however, either maddy itself should be compiled +with libpam support or a helper executable should be built and +installed into an appropriate directory. + +It is recommended to use builtin libpam support if you are using +PAM as an intermediate for authentication provider not directly +supported by maddy. + +If PAM authentication requires privileged access on the host system +(e.g. pam_unix.so aka /etc/shadow) then it is recommended to use +a privileged helper executable since maddy process itself won't +have access to it. + +## Built-in PAM support + +Binary artifacts provided for releases do not come with +libpam support. You should build maddy from source. + +See [here](../building-from-source) for detailed instructions. + +You should have libpam development files installed (`libpam-dev` +package on Ubuntu/Debian). + +Then add `--tags 'libpam'` to the build command: +``` +./build.sh --tags 'libpam' +``` + +Then you should be able to replace `local_authdb` implementation +in default configuration with `auth.pam`: +``` +auth.pam local_authdb { + use_helper no +} +``` + +## Helper executable + +TL;DR +``` +git clone https://github.com/foxcpp/maddy +cd maddy/cmd/maddy-pam-helper +gcc pam.c main.c -lpam -o maddy-pam-helper +``` + +Copy the resulting executable into /usr/lib/maddy/ and make +it setuid-root so it can read /etc/shadow (if that's necessary): +``` +chown root:maddy /usr/lib/maddy/maddy-pam-helper +chmod u+xs,g+x,o-x /usr/lib/maddy/maddy-pam-helper +``` + +Then you should be able to replace `local_authdb` implementation +in default configuration with `auth.pam`: +``` +auth.pam local_authdb { + use_helper yes +} +``` + +## Account names + +Since PAM does not use emails for authentication you should configure +maddy to either strip domain part when checking credentials or do not +use email when authenticating. + +See [Multiple domains configuration](/multiple-domains) for how to configure +authentication. + +## PAM service + +You should create a PAM configuration file for maddy to use. +Place it into /etc/pam.d/maddy. +Here is the minimal example using pam_unix (shadow database). +``` +#%PAM-1.0 +auth required pam_unix.so +account required pam_unix.so +``` + +Here is the configuration example you could use on Ubuntu +to use the authentication config system itself uses: +``` +#%PAM-1.0 + +@include common-auth +@include common-account +@include common-session +``` diff --git a/docs/tutorials/setting-up.md b/docs/tutorials/setting-up.md new file mode 100644 index 0000000..2ec8351 --- /dev/null +++ b/docs/tutorials/setting-up.md @@ -0,0 +1,259 @@ +# Installation & initial configuration + +This is the practical guide on how to set up a mail server using maddy for +personal use. It omits most of the technical details for brevity and just gives +you the minimal list of things you need to be aware of and what to do to make +stuff work. + +For purposes of clarity, these values are used in this tutorial as examples, +wherever you see them, you need to replace them with your actual values: + +- Domain: example.org +- MX domain (hostname): mx1.example.org +- IPv4 address: 10.2.3.4 +- IPv6 address: 2001:beef::1 + +## Getting a server + +Where to get a server to run maddy on is out of the scope of this article. Any +VPS (virtual private server) will work fine for small configurations. However, +there are a few things to keep in mind: + +- Make sure your provider does not block SMTP traffic (25 TCP port). Most VPS + providers don't do it, but some "cloud" providers (such as Google Cloud) do + it, so you can't host your mail there. + +- It is recommended to run your own DNS resolver with DNSSEC verification + enabled. + +## Installing maddy + +Your options are: + +* Pre-built tarball (Linux, amd64) + + Available on [GitHub](https://github.com/foxcpp/maddy/releases) or + [maddy.email/builds](https://maddy.email/builds/). + + The tarball includes maddy executable you can + copy into /usr/local/bin as well as systemd unit file you can + use on systemd-based distributions for automatic startup and service + supervision. You should also create "maddy" user and group. + See below for more detailed instructions. + +* Docker image (Linux, amd64) + + ``` + docker pull foxcpp/maddy:0.6 + ``` + + See [here](../../docker) for Docker-specific instructions. + +* Building from source + + See [here](../building-from-source) for instructions. + +* Arch Linux packages + + For Arch Linux users, `maddy` and `maddy-git` PKGBUILDs are available + in AUR. Additionally, binary packages are available in 3rd-party + repository at [https://maddy.email/archlinux/](https://maddy.email/archlinux/) + +## System configuration (systemd-based distribution) + +If you built maddy from source and used `./build.sh install` then +systemd unit files should be already installed. If you used +a pre-built tarball - copy `systemd/*.service` to `/etc/systemd/system` +manually. + +You need to reload service manager configuration to make service available: + +``` +systemctl daemon-reload +``` + +Additionally, you should create maddy user and group. Unlike most other +Linux mail servers, maddy never runs as root. + +``` +useradd -mrU -s /sbin/nologin -d /var/lib/maddy -c "maddy mail server" maddy +``` + +## Host name + domain + +Open /etc/maddy/maddy.conf with vim^W your favorite editor and change +the following lines to match your server name and domain you want to handle +mail for. +If you setup a very small mail server you can use example.org in both fields. +However, to easier a future migration of service, it's recommended to use a +separate DNS entry for that purpose. It's usually mx1.example.org, mx2, etc. +You can of course use another subdomain, for instance: smtp1.example.org. +An email failover server will become possible if you forward mx2.example.org +to another server (as long as you configure it to handle your domain). + +``` +$(hostname) = mx1.example.org +$(primary_domain) = example.org +``` + +If you want to handle multiple domains, you still need to designate +one as "primary". Add all other domains to the `local_domains` line: + +``` +$(local_domains) = $(primary_domain) example.com other.example.com +``` + +## TLS certificates + +One thing that can't be automagically configured is TLS certs. If you already +have them somewhere - use them, open /etc/maddy/maddy.conf and put the right +paths in. You need to make sure maddy can read them while running as +unprivileged user (maddy never runs as root, even during start-up), one way to +do so is to use ACLs (replace with your actual paths): +``` +$ sudo setfacl -R -m u:maddy:rX /etc/ssl/mx1.example.org.crt /etc/ssl/mx1.example.org.key +``` + +maddy reloads TLS certificates from disk once in a minute so it will notice +renewal. It is possible to force reload via `systemctl reload maddy` (or just +`killall -USR2 maddy`). + +### Let's Encrypt and certbot + +If you use certbot to manage your certificates, you can simply symlink +/etc/maddy/certs into /etc/letsencrypt/live. maddy will pick the right +certificate depending on the domain you specified during installation. + +You still need to make keys readable for maddy, though: +``` +$ sudo setfacl -R -m u:maddy:rX /etc/letsencrypt/{live,archive} +``` + +### ACME.sh + +If you use acme.sh to manage your certificates, you could simply run: + +``` +mkdir -p /etc/maddy/certs/mx1.example.org +acme.sh --force --install-cert -d mx1.example.org \ + --key-file /etc/maddy/certs/mx1.example.org/privkey.pem \ + --fullchain-file /etc/maddy/certs/mx1.example.org/fullchain.pem +``` + +## First run + +``` +systemctl start maddy +``` + +The daemon should be running now, except that it is useless because we haven't +configured DNS records. + +## DNS records + +How it is configured depends on your DNS provider (or server, if you run your +own). Here is how your DNS zone should look like: +``` +; Basic domain->IP records, you probably already have them. +example.org. A 10.2.3.4 +example.org. AAAA 2001:beef::1 + +; It says that "server mx1.example.org is handling messages for example.org". +example.org. MX 10 mx1.example.org. +; Of course, mx1 should have A/AAAA entry as well: +mx1.example.org. A 10.2.3.4 +mx1.example.org. AAAA 2001:beef::1 + +; Use SPF to say that the servers in "MX" above are allowed to send email +; for this domain, and nobody else. +example.org. TXT "v=spf1 mx ~all" +; It is recommended to server SPF record for both domain and MX hostname +mx1.example.org. TXT "v=spf1 a ~all" + +; Opt-in into DMARC with permissive policy and request reports about broken +; messages. +_dmarc.example.org. TXT "v=DMARC1; p=quarantine; ruf=mailto:postmaster@example.org" + +; Mark domain as MTA-STS compatible (see the next section) +; and request reports about failures to be sent to postmaster@example.org +_mta-sts.example.org. TXT "v=STSv1; id=1" +_smtp._tls.example.org. TXT "v=TLSRPTv1;rua=mailto:postmaster@example.org" +``` + +And the last one, DKIM key, is a bit tricky. maddy generated a key for you on +the first start-up. You can find it in +/var/lib/maddy/dkim_keys/example.org_default.dns. You need to put it in a TXT +record for `default._domainkey.example.org.` domain, like that: +``` +default._domainkey.example.org. TXT "v=DKIM1; k=ed25519; p=nAcUUozPlhc4VPhp7hZl+owES7j7OlEv0laaDEDBAqg=" +``` + +## MTA-STS and DANE + +By default SMTP is not protected against active attacks. MTA-STS policy tells +compatible senders to always use properly authenticated TLS when talking to +your server, offering a simple-to-deploy way to protect your server against +MitM attacks on port 25. + +Basically, you to create a file with following contents and make it available +at https://mta-sts.example.org/.well-known/mta-sts.txt: +``` +version: STSv1 +mode: enforce +max_age: 604800 +mx: mx1.example.org +``` + +**Note**: mx1.example.org in the file is your MX hostname, In a simple configuration, +it will be the same as your hostname example.org. +In a more complex setups, you would have multiple MX servers - add them all once +per line, like that: + +``` +mx: mx1.example.org +mx: mx2.example.org +``` + +It is also recommended to set a TLSA (DANE) record. +Use https://www.huque.com/bin/gen_tlsa to generate one. +Set port to 25, Transport Protocol to "tcp" and Domain Name to **the MX hostname**. +Example of a valid record: +``` +_25._tcp.mx1.example.org. TLSA 3 1 1 7f59d873a70e224b184c95a4eb54caa9621e47d48b4a25d312d83d96e3498238 +``` + +## User accounts and maddy command + +A mail server is useless without mailboxes, right? Unlike software like postfix +and dovecot, maddy uses "virtual users" by default, meaning it does not care or +know about system users. + +IMAP mailboxes ("accounts") and authentication credentials are kept separate. + +To register user credentials, use `maddy creds create` command. +Like that: +``` +$ maddy creds create postmaster@example.org +``` + +Note the username is a e-mail address. This is required as username is used to +authorize IMAP and SMTP access (unless you configure custom mappings, not +described here). + +After registering the user credentials, you also need to create a local +storage account: +``` +$ maddy imap-acct create postmaster@example.org +``` + +Note: to run `maddy` CLI commands, your user should be in the `maddy` +group. Alternatively, just use `sudo -u maddy`. + +That is it. Now you have your first e-mail address. when authenticating using +your e-mail client, do not forget the username is "postmaster@example.org", not +just "postmaster". + +You may find running `maddy creds --help` and `maddy imap-acct --help` +useful to learn about other commands. Note that IMAP accounts and credentials +are managed separately yet usernames should match by default for things to +work. diff --git a/docs/upgrading.md b/docs/upgrading.md new file mode 100644 index 0000000..88fe4c7 --- /dev/null +++ b/docs/upgrading.md @@ -0,0 +1,117 @@ +# Upgrading from older maddy versions + +It is generally possible to just install latest version (e.g. using build.sh +script) over the existing installation. + +It is recommended to backup state directory (usually /var/lib/maddy for Linux) +before doing so. The new server version may automatically convert DB files in a +way that will make them unreadable by older versions. + +Specific instructions for upgrading between versions with incompatible changes +are documented on this page below. + +## Incompatible version migration + +## 0.2 -> 0.3 + +0.3 includes a significant change to the authentication code that makes it +completely independent of IMAP index. This means 0.2 "unified" database cannot +be used in 0.3 and auto-migration is not possible. Additionally, the way +passwords are hashed is changed, meaning that after migration passwords will +need to be reset. + +**Migration utility is SQLite-specific, if you need one that works for +Postgres - reach out at the IRC channel.** + +1. Make sure the server is not running. + +``` +systemctl stop maddy +``` + +2. Take a backup of `imapsql.db*` files in state directory (/var/lib/maddy). + +``` +mkdir backup +cp /var/lib/maddy/imapsql.db* backup/ +``` + +3. Compile migration utility: + +``` +git clone https://github.com/foxcpp/maddy.git +cd maddy/ +git checkout v0.3.0 +cd cmd/migrate-db-0.2 +go build +``` + +4. Run compiled binary: + +``` +./migrate-db-0.2 /var/lib/maddy/imapsql.db +``` + +5. Open maddy.conf and make following changes: + +Remove `local_authdb` name from imapsql configuration block: +``` +imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db +} +``` + +Add `local_authdb` configuration block using `pass_table` module: + +``` +pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } +} +``` + +6. Use `maddy creds create ACCOUNT_NAME` to add credentials to `pass_table` + store. + +7. Start the server back. + +``` +systemctl start maddy +``` + +## 0.1 -> 0.2 + +0.2 requires several changes in configuration file. + +Change +``` +sql local_mailboxes local_authdb { +``` +to +``` +imapsql local_mailboxes local_authdb { +``` + +Replace +``` +replace_rcpt postmaster postmaster@$(primary_domain) +``` +with +``` +replace_rcpt static { + entry postmaster postmaster@$(primary_domain) +} +``` +and + +``` +replace_rcpt "(.+)\+(.+)@(.+)" "$1@$3" +``` +with +``` +replace_rcpt regexp "(.+)\+(.+)@(.+)" "$1@$3" +``` diff --git a/framework/address/doc.go b/framework/address/doc.go new file mode 100644 index 0000000..3478a3d --- /dev/null +++ b/framework/address/doc.go @@ -0,0 +1,21 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package address provides utilities for parsing +// and validation of RFC 2821 addresses. +package address diff --git a/framework/address/norm.go b/framework/address/norm.go new file mode 100644 index 0000000..6998e66 --- /dev/null +++ b/framework/address/norm.go @@ -0,0 +1,169 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package address + +import ( + "fmt" + "strings" + "unicode/utf8" + + "github.com/foxcpp/maddy/framework/dns" + "golang.org/x/net/idna" + "golang.org/x/text/secure/precis" + "golang.org/x/text/unicode/norm" +) + +// ForLookup transforms the local-part of the address into a canonical form +// usable for map lookups or direct comparisons. +// +// If Equal(addr1, addr2) == true, then ForLookup(addr1) == ForLookup(addr2). +// +// On error, case-folded addr is also returned. +func ForLookup(addr string) (string, error) { + if addr == "" { // Null return-path case. + return "", nil + } + + mbox, domain, err := Split(addr) + if err != nil { + return strings.ToLower(addr), err + } + + if domain != "" { + domain, err = dns.ForLookup(domain) + if err != nil { + return strings.ToLower(addr), err + } + } + + mbox = strings.ToLower(norm.NFC.String(mbox)) + + if domain == "" { + return mbox, nil + } + + return mbox + "@" + domain, nil +} + +// CleanDomain returns the address with the domain part converted into its canonical form. +// +// More specifically, converts the domain part of the address to U-labels, +// normalizes it to NFC and then case-folds it. +// +// Original value is also returned on the error. +func CleanDomain(addr string) (string, error) { + if addr == "" { // Null return-path + return "", nil + } + + mbox, domain, err := Split(addr) + if err != nil { + return addr, err + } + + uDomain, err := idna.ToUnicode(domain) + if err != nil { + return addr, err + } + uDomain = strings.ToLower(norm.NFC.String(uDomain)) + + if domain == "" { + return mbox, nil + } + + return mbox + "@" + uDomain, nil +} + +// Equal reports whether addr1 and addr2 are considered to be +// case-insensitively equivalent. +// +// The equivalence is defined to be the conjunction of IDN label equivalence +// for the domain part and canonical equivalence* of the local-part converted +// to lower case. +// +// * IDN label equivalence is defined by RFC 5890 Section 2.3.2.4. +// ** Canonical equivalence is defined by UAX #15. +// +// Equivalence for malformed addresses is defined using regular byte-string +// comparison with case-folding applied. +func Equal(addr1, addr2 string) bool { + // Short circuit. If they are bit-equivalent, then they are also canonically + // equivalent. + if addr1 == addr2 { + return true + } + + uAddr1, _ := ForLookup(addr1) + uAddr2, _ := ForLookup(addr2) + return uAddr1 == uAddr2 +} + +func IsASCII(s string) bool { + for _, ch := range s { + if ch > utf8.RuneSelf { + return false + } + } + return true +} + +func FQDNDomain(addr string) string { + if strings.HasSuffix(addr, ".") { + return addr + } + return addr + "." +} + +// PRECISFold applies UsernameCaseMapped to the local part and dns.ForLookup +// to domain part of the address. +func PRECISFold(addr string) (string, error) { + return precisEmail(addr, precis.UsernameCaseMapped) +} + +// PRECIS applies UsernameCasePreserved to the local part and dns.ForLookup +// to domain part of the address. +func PRECIS(addr string) (string, error) { + return precisEmail(addr, precis.UsernameCasePreserved) +} + +func precisEmail(addr string, profile *precis.Profile) (string, error) { + mbox, domain, err := Split(addr) + if err != nil { + return "", fmt.Errorf("address: precis: %w", err) + } + + // PRECISFold is not included in the regular address.ForLookup since it reduces + // the range of valid addresses to a subset of actually valid values. + // PRECISFold is a matter of our own local policy, not a general rule for all + // email addresses. + + // Side note: For used profiles, there is no practical difference between + // CompareKey and String. + mbox, err = profile.CompareKey(mbox) + if err != nil { + return "", fmt.Errorf("address: precis: %w", err) + } + + domain, err = dns.ForLookup(domain) + if err != nil { + return "", fmt.Errorf("address: precis: %w", err) + } + + return mbox + "@" + domain, nil +} diff --git a/framework/address/norm_test.go b/framework/address/norm_test.go new file mode 100644 index 0000000..1b6d511 --- /dev/null +++ b/framework/address/norm_test.go @@ -0,0 +1,88 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package address + +import ( + "testing" +) + +func addrFuncTest(t *testing.T, f func(string) (string, error)) func(in, wantOut string, fail bool) { + return func(in, wantOut string, fail bool) { + t.Helper() + + out, err := f(in) + if err != nil { + if !fail { + t.Errorf("Expected failure, got none") + } + } + if out != wantOut { + t.Errorf("Wrong result: want '%s', got '%s'", wantOut, out) + } + } +} + +func TestForLookup(t *testing.T) { + test := addrFuncTest(t, ForLookup) + test("test@example.org", "test@example.org", false) + test("E\u0301@example.org", "\u00E9@example.org", false) + test("test@EXAMPLE.org", "test@example.org", false) + test("test@xn--e1aybc.example.org", "test@тест.example.org", false) + test("TEST@xn--99999999999.example.org", "test@xn--99999999999.example.org", true) + test("tESt@", "test@", true) + test("postmaster", "postmaster", false) +} + +func TestCleanDomain(t *testing.T) { + test := addrFuncTest(t, CleanDomain) + test("test@example.org", "test@example.org", false) + test("whateveR@example.org", "whateveR@example.org", false) + test("E\u0301@example.org", "E\u0301@example.org", false) + test("test@EXAMPLE.org", "test@example.org", false) + test("test@xn--e1aybc.example.org", "test@тест.example.org", false) + test("TEST@xn--99999999999.example.org", "TEST@xn--99999999999.example.org", true) + test("tESt@", "tESt@", true) + test("postmaster", "postmaster", false) +} + +func TestEqual(t *testing.T) { + test := func(in1, in2 string, wantEq bool) { + eq := Equal(in1, in2) + if eq != wantEq { + t.Errorf("Want Equal(%s, %s) == %v, got %v", in1, in2, wantEq, eq) + } + } + + test("test@example.org", "test@example.org", true) + test("test2@example.org", "test@example.org", false) + test("TEST2@example.org", "TesT2@example.org", true) + test("E\u0301@example.org", "\u00E9@example.org", true) + test("test@тест.example.org", "test@xn--e1aybc.example.org", true) + test("test@xn--999999999999999.example.org", "test@xn--999999999999999.example.org", true) + test("test@xn--999999999999.example.org", "test@xn--999999999999999.example.org", false) +} + +func TestIsASCII(t *testing.T) { + if !IsASCII("hello") { + t.Errorf("'hello' is ASCII") + } + if IsASCII("тест") { + t.Errorf("'тест' is non-ASCII") + } +} diff --git a/framework/address/rfc6531.go b/framework/address/rfc6531.go new file mode 100644 index 0000000..dfa5420 --- /dev/null +++ b/framework/address/rfc6531.go @@ -0,0 +1,85 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package address + +import ( + "errors" + + "golang.org/x/net/idna" + "golang.org/x/text/unicode/norm" +) + +var ErrUnicodeMailbox = errors.New("address: cannot convert the Unicode local-part to the ACE form") + +// ToASCII converts the domain part of the email address to the A-label form and +// fails with ErrUnicodeMailbox if the local-part contains non-ASCII characters. +func ToASCII(addr string) (string, error) { + mbox, domain, err := Split(addr) + if err != nil { + return addr, err + } + + for _, ch := range mbox { + if ch > 128 { + return addr, ErrUnicodeMailbox + } + } + + if domain == "" { + return mbox, nil + } + + aDomain, err := idna.ToASCII(domain) + if err != nil { + return addr, err + } + + return mbox + "@" + aDomain, nil +} + +// ToUnicode converts the domain part of the email address to the U-label form. +func ToUnicode(addr string) (string, error) { + mbox, domain, err := Split(addr) + if err != nil { + return norm.NFC.String(addr), err + } + + if domain == "" { + return mbox, nil + } + + uDomain, err := idna.ToUnicode(domain) + if err != nil { + return norm.NFC.String(addr), err + } + + return mbox + "@" + norm.NFC.String(uDomain), nil +} + +// SelectIDNA is a convenience function for conversion of domains in the email +// addresses to/from the Punycode form. +// +// ulabel=true => ToUnicode is used. +// ulabel=false => ToASCII is used. +func SelectIDNA(ulabel bool, addr string) (string, error) { + if ulabel { + return ToUnicode(addr) + } + return ToASCII(addr) +} diff --git a/framework/address/rfc6531_test.go b/framework/address/rfc6531_test.go new file mode 100644 index 0000000..f6cec25 --- /dev/null +++ b/framework/address/rfc6531_test.go @@ -0,0 +1,41 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package address + +import ( + "strings" + "testing" +) + +func TestToASCII(t *testing.T) { + test := addrFuncTest(t, ToASCII) + test("test@тест.example.org", "test@xn--e1aybc.example.org", false) + test("test@org."+strings.Repeat("x", 65535)+"\uFF00", "test@org."+strings.Repeat("x", 65535)+"\uFF00", true) + test("тест@example.org", "тест@example.org", true) + test("postmaster", "postmaster", false) + test("postmaster@", "postmaster@", true) +} + +func TestToUnicode(t *testing.T) { + test := addrFuncTest(t, ToUnicode) + test("test@xn--e1aybc.example.org", "test@тест.example.org", false) + test("test@xn--9999999999999999999a.org", "test@xn--9999999999999999999a.org", true) + test("postmaster", "postmaster", false) + test("postmaster@", "postmaster@", true) +} diff --git a/framework/address/split.go b/framework/address/split.go new file mode 100644 index 0000000..88b8d51 --- /dev/null +++ b/framework/address/split.go @@ -0,0 +1,132 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package address + +import ( + "errors" + "strings" +) + +// Split splits a email address (as defined by RFC 5321 as a forward-path +// token) into local part (mailbox) and domain. +// +// Note that definition of the forward-path token includes the special +// postmaster address without the domain part. Split will return domain == "" +// in this case. +// +// Split does almost no sanity checks on the input and is intentionally naive. +// If this is a concern, ValidMailbox and ValidDomain should be used on the +// output. +func Split(addr string) (mailbox, domain string, err error) { + if strings.EqualFold(addr, "postmaster") { + return addr, "", nil + } + + indx := strings.LastIndexByte(addr, '@') + if indx == -1 { + return "", "", errors.New("address: missing at-sign") + } + mailbox = addr[:indx] + domain = addr[indx+1:] + if mailbox == "" { + return "", "", errors.New("address: empty local-part") + } + if domain == "" { + return "", "", errors.New("address: empty domain") + } + return +} + +// UnquoteMbox undoes escaping and quoting of the local-part. That is, for +// local-part `"test\" @ test"` it will return `test" @test`. +func UnquoteMbox(mbox string) (string, error) { + var ( + quoted bool + escaped bool + terminatedQuote bool + mailboxB strings.Builder + ) + for _, ch := range mbox { + if terminatedQuote { + return "", errors.New("address: closing quote should be right before at-sign") + } + + switch ch { + case '"': + if !escaped { + quoted = !quoted + if !quoted { + terminatedQuote = true + } + continue + } + case '\\': + if !escaped { + if !quoted { + return "", errors.New("address: escapes are allowed only in quoted strings") + } + escaped = true + continue + } + case '@': + if !quoted { + return "", errors.New("address: extra at-sign in non-quoted local-part") + } + } + + escaped = false + + mailboxB.WriteRune(ch) + } + + if mailboxB.Len() == 0 { + return "", errors.New("address: empty local part") + } + + return mailboxB.String(), nil +} + +// "specials" from RFC5322 grammar with dot removed (it is defined in grammar separately, for some reason) +var mboxSpecial = map[rune]struct{}{ + '(': {}, ')': {}, '<': {}, '>': {}, + '[': {}, ']': {}, ':': {}, ';': {}, + '@': {}, '\\': {}, ',': {}, + '"': {}, ' ': {}, +} + +func QuoteMbox(mbox string) string { + var mailboxEsc strings.Builder + mailboxEsc.Grow(len(mbox)) + quoted := false + for _, ch := range mbox { + if _, ok := mboxSpecial[ch]; ok { + if ch == '\\' || ch == '"' { + mailboxEsc.WriteRune('\\') + } + mailboxEsc.WriteRune(ch) + quoted = true + } else { + mailboxEsc.WriteRune(ch) + } + } + if quoted { + return `"` + mailboxEsc.String() + `"` + } + return mbox +} diff --git a/framework/address/split_test.go b/framework/address/split_test.go new file mode 100644 index 0000000..b5a8df5 --- /dev/null +++ b/framework/address/split_test.go @@ -0,0 +1,110 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package address + +import ( + "testing" +) + +func TestSplit(t *testing.T) { + test := func(addr, mbox, domain string, fail bool) { + t.Helper() + + actualMbox, actualDomain, err := Split(addr) + if err != nil && !fail { + t.Errorf("%s: unexpected error: %v", addr, err) + return + } + if err == nil && fail { + t.Errorf("%s: expected error, got %s, %s", addr, actualMbox, actualDomain) + return + } + + if actualMbox != mbox { + t.Errorf("%s: wrong local part, want %s, got %s", addr, mbox, actualMbox) + } + if actualDomain != domain { + t.Errorf("%s: wrong domain part, want %s, got %s", addr, domain, actualDomain) + } + } + + test("simple@example.org", "simple", "example.org", false) + test("simple@[1.2.3.4]", "simple", "[1.2.3.4]", false) + test("simple@[IPv6:beef::1]", "simple", "[IPv6:beef::1]", false) + test("@example.org", "", "", true) + test("@", "", "", true) + test("no-domain@", "", "", true) + test("@no-local-part", "", "", true) + + // Not a valid address, but a special value for SMTP + // should be handled separately where necessary. + test("", "", "", true) + + // A special SMTP value too, but permitted now. + test("postmaster", "postmaster", "", false) +} + +func TestUnquoteMbox(t *testing.T) { + test := func(inputMbox, expectedMbox string, fail bool) { + t.Helper() + + actualMbox, err := UnquoteMbox(inputMbox) + if err != nil && !fail { + t.Errorf("unexpected error: %v", err) + return + } + if err == nil && fail { + t.Errorf("expected error, got %s", actualMbox) + return + } + + if actualMbox != expectedMbox { + t.Errorf("wrong local part, want %s, got %s", actualMbox, actualMbox) + } + } + + test(`no\@no`, "", true) + test("no@no", "", true) + test(`no\"no`, "", true) + test(`"no\"no"`, `no"no`, false) + test(`"no@no"`, `no@no`, false) + test(`"no no"`, `no no`, false) + test(`"no\\no"`, `no\no`, false) + test(`"no"no`, "", true) + test(`postmaster`, "postmaster", false) + test(`foo`, "foo", false) +} + +func TestQuoteMbox(t *testing.T) { + test := func(inputMbox, expectedMbox string) { + t.Helper() + + actualMbox := QuoteMbox(inputMbox) + if actualMbox != expectedMbox { + t.Errorf("wrong local part, want %s, got %s", actualMbox, actualMbox) + } + } + + test(`no"no`, `"no\"no"`) + test(`no@no`, `"no@no"`) + test(`no no`, `"no no"`) + test(`no\no`, `"no\\no"`) + test("postmaster", `postmaster`) + test("foo", `foo`) +} diff --git a/framework/address/validation.go b/framework/address/validation.go new file mode 100644 index 0000000..a165adc --- /dev/null +++ b/framework/address/validation.go @@ -0,0 +1,138 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package address + +import ( + "strings" + + "golang.org/x/net/idna" +) + +/* +Rules for validation are subset of rules listed here: +https://emailregex.com/email-validation-summary/ +*/ + +// Valid checks whether ths string is valid as a email address as defined by +// RFC 5321. +func Valid(addr string) bool { + if len(addr) > 320 { // RFC 3696 says it's 320, not 255. + return false + } + + mbox, domain, err := Split(addr) + if err != nil { + return false + } + + // The only case where this can be true is "postmaster". + // So allow it. + if domain == "" { + return true + } + + return ValidMailboxName(mbox) && ValidDomain(domain) +} + +var validGraphic = map[rune]bool{ + '!': true, '#': true, + '$': true, '%': true, + '&': true, '\'': true, + '*': true, '+': true, + '-': true, '/': true, + '=': true, '?': true, + '^': true, '_': true, + '`': true, '{': true, + '|': true, '}': true, + '~': true, '.': true, +} + +// ValidMailboxName checks whether the specified string is a valid mailbox-name +// element of e-mail address (left part of it, before at-sign). +func ValidMailboxName(mbox string) bool { + if strings.HasPrefix(mbox, `"`) { + raw, err := UnquoteMbox(mbox) + if err != nil { + return false + } + + // Inside quotes, any ASCII graphic and space is allowed. + // Additionally, RFC 6531 extends that to allow any Unicode (UTF-8). + for _, ch := range raw { + if ch < ' ' || ch == 0x7F /* DEL */ { + // ASCII control characters. + return false + } + } + return true + } + + // Without quotes, limited set of ASCII graphics is allowed + ASCII + // alphanumeric characters. + // RFC 6531 extends that to allow any Unicode (UTF-8). + for _, ch := range mbox { + if validGraphic[ch] { + continue + } + if ch >= '0' && ch <= '9' { + continue + } + if ch >= 'A' && ch <= 'Z' { + continue + } + if ch >= 'a' && ch <= 'z' { + continue + } + if ch > 0x7F { // Unicode + continue + } + + return false + } + + return true +} + +// ValidDomain checks whether the specified string is a valid DNS domain. +func ValidDomain(domain string) bool { + if len(domain) > 255 || len(domain) == 0 { + return false + } + if strings.HasPrefix(domain, ".") { + return false + } + if strings.Contains(domain, "..") { + return false + } + + // Length checks are to be applied to A-labels form. + // maddy uses U-labels representation across the code (for lookups, etc). + domainASCII, err := idna.ToASCII(domain) + if err != nil { + return false + } + labels := strings.Split(domainASCII, ".") + for _, label := range labels { + if len(label) > 64 { + return false + } + } + + return true +} diff --git a/framework/address/validation_test.go b/framework/address/validation_test.go new file mode 100644 index 0000000..fdac834 --- /dev/null +++ b/framework/address/validation_test.go @@ -0,0 +1,33 @@ +package address_test + +import ( + "strings" + "testing" + + "github.com/foxcpp/maddy/framework/address" +) + +func TestValidMailboxName(t *testing.T) { + if !address.ValidMailboxName("caddy.bug") { + t.Error("caddy.bug should be valid mailbox name") + } +} + +func TestValidDomain(t *testing.T) { + for _, c := range []struct { + Domain string + Valid bool + }{ + {Domain: "maddy.email", Valid: true}, + {Domain: "", Valid: false}, + {Domain: "maddy.email.", Valid: true}, + {Domain: "..", Valid: false}, + {Domain: strings.Repeat("a", 256), Valid: false}, + {Domain: "äõäoaõoäaõaäõaoäaoaäõoaäooaoaoiuaiauäõiuüõaõäiauõaaa.tld", Valid: true}, // https://github.com/foxcpp/maddy/issues/554 + {Domain: "xn--oaoaaaoaoaoaooaoaoiuaiauiuaiauaaa-f1cadccdcmd01eddchqcbe07a.tld", Valid: true}, // https://github.com/foxcpp/maddy/issues/554 + } { + if actual := address.ValidDomain(c.Domain); actual != c.Valid { + t.Errorf("expected domain %v to be valid=%v, but got %v", c.Domain, c.Valid, actual) + } + } +} diff --git a/framework/buffer/buffer.go b/framework/buffer/buffer.go new file mode 100644 index 0000000..e386046 --- /dev/null +++ b/framework/buffer/buffer.go @@ -0,0 +1,60 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// The buffer package provides utilities for temporary storage (buffering) +// of large blobs. +package buffer + +import ( + "io" +) + +// Buffer interface represents abstract temporary storage for blobs. +// +// The Buffer storage is assumed to be immutable. If any modifications +// are made - new storage location should be used for them. +// This is important to ensure goroutine-safety. +// +// Since Buffer objects require a careful management of lifetimes, here +// is the convention: Its always creator responsibility to call Remove after +// Buffer is no longer used. If Buffer object is passed to a function - it is not +// guaranteed to be valid after this function returns. If function needs to preserve +// the storage contents, it should "re-buffer" it either by reading entire blob +// and storing it somewhere or applying implementation-specific methods (for example, +// the FileBuffer storage may be "re-buffered" by hard-linking the underlying file). +type Buffer interface { + // Open creates new Reader reading from the underlying storage. + Open() (io.ReadCloser, error) + + // Len reports the length of the stored blob. + // + // Notably, it indicates the amount of bytes that can be read from the + // newly created Reader without hiting io.EOF. + Len() int + + // Remove discards buffered body and releases all associated resources. + // + // Multiple Buffer objects may refer to the same underlying storage. + // In this case, care should be taken to ensure that Remove is called + // only once since it will discard the shared storage and invalidate + // all Buffer objects using it. + // + // Readers previously created using Open can still be used, but + // new ones can't be created. + Remove() error +} diff --git a/framework/buffer/bytesreader.go b/framework/buffer/bytesreader.go new file mode 100644 index 0000000..365a75f --- /dev/null +++ b/framework/buffer/bytesreader.go @@ -0,0 +1,61 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package buffer + +import ( + "bytes" +) + +// BytesReader is a wrapper for bytes.Reader that stores the original []byte +// value and allows to retrieve it. +// +// It is meant for passing to libraries that expect a io.Reader +// but apply certain optimizations when the Reader implements +// Bytes() interface. +type BytesReader struct { + *bytes.Reader + value []byte +} + +// Bytes returns the unread portion of underlying slice used to construct +// BytesReader. +func (br BytesReader) Bytes() []byte { + return br.value[int(br.Size())-br.Len():] +} + +// Copy returns the BytesReader reading from the same slice as br at the same +// position. +func (br BytesReader) Copy() BytesReader { + return NewBytesReader(br.Bytes()) +} + +// Close is a dummy method for implementation of io.Closer so BytesReader can +// be used in MemoryBuffer directly. +func (br BytesReader) Close() error { + return nil +} + +func NewBytesReader(b []byte) BytesReader { + // BytesReader and not *BytesReader because BytesReader already wraps two + // pointers and double indirection would be pointless. + return BytesReader{ + Reader: bytes.NewReader(b), + value: b, + } +} diff --git a/framework/buffer/file.go b/framework/buffer/file.go new file mode 100644 index 0000000..0025984 --- /dev/null +++ b/framework/buffer/file.go @@ -0,0 +1,85 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package buffer + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" +) + +// FileBuffer implements Buffer interface using file system. +type FileBuffer struct { + Path string + + // LenHint is the size of the stored blob. It can + // be set to avoid the need to call os.Stat in the + // Len() method. + LenHint int +} + +func (fb FileBuffer) Open() (io.ReadCloser, error) { + return os.Open(fb.Path) +} + +func (fb FileBuffer) Len() int { + if fb.LenHint != 0 { + return fb.LenHint + } + + info, err := os.Stat(fb.Path) + if err != nil { + // Any access to the file will probably fail too. So we can't return a + // sensible value. + return 0 + } + + return int(info.Size()) +} + +func (fb FileBuffer) Remove() error { + return os.Remove(fb.Path) +} + +// BufferInFile is a convenience function which creates FileBuffer with underlying +// file created in the specified directory with the random name. +func BufferInFile(r io.Reader, dir string) (Buffer, error) { + // It is assumed that PRNG is initialized somewhere during program startup. + nameBytes := make([]byte, 32) + _, err := rand.Read(nameBytes) + if err != nil { + return nil, fmt.Errorf("buffer: failed to generate randomness for file name: %v", err) + } + path := filepath.Join(dir, hex.EncodeToString(nameBytes)) + f, err := os.Create(path) + if err != nil { + return nil, fmt.Errorf("buffer: failed to create file: %v", err) + } + if _, err = io.Copy(f, r); err != nil { + return nil, fmt.Errorf("buffer: failed to write file: %v", err) + } + if err := f.Close(); err != nil { + return nil, fmt.Errorf("buffer: failed to close file: %v", err) + } + + return FileBuffer{Path: path}, nil +} diff --git a/framework/buffer/memory.go b/framework/buffer/memory.go new file mode 100644 index 0000000..dafd677 --- /dev/null +++ b/framework/buffer/memory.go @@ -0,0 +1,50 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package buffer + +import ( + "io" +) + +// MemoryBuffer implements Buffer interface using byte slice. +type MemoryBuffer struct { + Slice []byte +} + +func (mb MemoryBuffer) Open() (io.ReadCloser, error) { + return NewBytesReader(mb.Slice), nil +} + +func (mb MemoryBuffer) Len() int { + return len(mb.Slice) +} + +func (mb MemoryBuffer) Remove() error { + return nil +} + +// BufferInMemory is a convenience function which creates MemoryBuffer with +// contents of the passed io.Reader. +func BufferInMemory(r io.Reader) (Buffer, error) { + blob, err := io.ReadAll(r) + if err != nil { + return nil, err + } + return MemoryBuffer{Slice: blob}, nil +} diff --git a/framework/cfgparser/env.go b/framework/cfgparser/env.go new file mode 100644 index 0000000..c7ad03c --- /dev/null +++ b/framework/cfgparser/env.go @@ -0,0 +1,67 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package parser + +import ( + "os" + "regexp" + "strings" +) + +func expandEnvironment(nodes []Node) []Node { + // If nodes is nil - don't replace with empty slice, as nil indicates "no + // block". + if nodes == nil { + return nil + } + + replacer := buildEnvReplacer() + newNodes := make([]Node, 0, len(nodes)) + for _, node := range nodes { + node.Name = removeUnexpandedEnvvars(replacer.Replace(node.Name)) + newArgs := make([]string, 0, len(node.Args)) + for _, arg := range node.Args { + newArgs = append(newArgs, removeUnexpandedEnvvars(replacer.Replace(arg))) + } + node.Args = newArgs + node.Children = expandEnvironment(node.Children) + newNodes = append(newNodes, node) + } + return newNodes +} + +var unixEnvvarRe = regexp.MustCompile(`{env:([^\$]+)}`) + +func removeUnexpandedEnvvars(s string) string { + s = unixEnvvarRe.ReplaceAllString(s, "") + return s +} + +func buildEnvReplacer() *strings.Replacer { + env := os.Environ() + pairs := make([]string, 0, len(env)*4) + for _, entry := range env { + parts := strings.SplitN(entry, "=", 2) + key := parts[0] + value := parts[1] + + pairs = append(pairs, "{env:"+key+"}", value) + } + return strings.NewReplacer(pairs...) +} diff --git a/framework/cfgparser/imports.go b/framework/cfgparser/imports.go new file mode 100644 index 0000000..8078dd8 --- /dev/null +++ b/framework/cfgparser/imports.go @@ -0,0 +1,176 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package parser + +import ( + "os" + "path/filepath" + "regexp" + "strings" +) + +func (ctx *parseContext) expandImports(node Node, expansionDepth int) (Node, error) { + // Leave nil value as is because it is used as non-existent block indicator + // (vs empty slice - empty block). + if node.Children == nil { + return node, nil + } + + newChildrens := make([]Node, 0, len(node.Children)) + containsImports := false + for _, child := range node.Children { + child, err := ctx.expandImports(child, expansionDepth+1) + if err != nil { + return node, err + } + + if child.Name == "import" { + // We check it here instead of function start so we can + // use line information from import directive that is likely + // caused this error. + if expansionDepth > 255 { + return node, NodeErr(child, "hit import expansion limit") + } + + containsImports = true + if len(child.Args) != 1 { + return node, ctx.Err("import directive requires exactly 1 argument") + } + + subtree, err := ctx.resolveImport(child, child.Args[0], expansionDepth) + if err != nil { + return node, err + } + + newChildrens = append(newChildrens, subtree...) + } else { + newChildrens = append(newChildrens, child) + } + } + node.Children = newChildrens + + // We need to do another pass to expand any imports added by snippets we + // just expanded. + if containsImports { + return ctx.expandImports(node, expansionDepth+1) + } + + return node, nil +} + +func (ctx *parseContext) resolveImport(node Node, name string, expansionDepth int) ([]Node, error) { + if subtree, ok := ctx.snippets[name]; ok { + return subtree, nil + } + + file := name + if !filepath.IsAbs(name) { + file = filepath.Join(filepath.Dir(ctx.fileLocation), name) + } + src, err := os.Open(file) + if err != nil { + if os.IsNotExist(err) { + src, err = os.Open(file + ".conf") + if err != nil { + if os.IsNotExist(err) { + return nil, NodeErr(node, "unknown import: %s", name) + } + return nil, err + } + } else { + return nil, err + } + } + nodes, snips, macros, err := readTree(src, file, expansionDepth+1) + if err != nil { + return nodes, err + } + for k, v := range snips { + ctx.snippets[k] = v + } + for k, v := range macros { + ctx.macros[k] = v + } + + return nodes, nil +} + +func (ctx *parseContext) expandMacros(node *Node) error { + if strings.HasPrefix(node.Name, "$(") && strings.HasSuffix(node.Name, ")") { + return ctx.Err("can't use macro argument as directive name") + } + + newArgs := make([]string, 0, len(node.Args)) + for _, arg := range node.Args { + if !strings.HasPrefix(arg, "$(") || !strings.HasSuffix(arg, ")") { + if strings.Contains(arg, "$(") && strings.Contains(arg, ")") { + var err error + arg, err = ctx.expandSingleValueMacro(arg) + if err != nil { + return err + } + } + + newArgs = append(newArgs, arg) + continue + } + + macroName := arg[2 : len(arg)-1] + replacement, ok := ctx.macros[macroName] + if !ok { + // Undefined macros are expanded to zero arguments. + continue + } + + newArgs = append(newArgs, replacement...) + } + node.Args = newArgs + + if node.Children != nil { + for i := range node.Children { + if err := ctx.expandMacros(&node.Children[i]); err != nil { + return err + } + } + } + + return nil +} + +var macroRe = regexp.MustCompile(`\$\(([^\$]+)\)`) + +func (ctx *parseContext) expandSingleValueMacro(arg string) (string, error) { + matches := macroRe.FindAllStringSubmatch(arg, -1) + for _, match := range matches { + macroName := match[1] + if len(ctx.macros[macroName]) > 1 { + return "", ctx.Err("can't expand macro with multiple arguments inside a string") + } + + var value string + if ctx.macros[macroName] != nil { + // Macros have at least one argument. + value = ctx.macros[macroName][0] + } + + arg = strings.Replace(arg, "$("+macroName+")", value, -1) + } + + return arg, nil +} diff --git a/framework/cfgparser/parse.go b/framework/cfgparser/parse.go new file mode 100644 index 0000000..aed01e3 --- /dev/null +++ b/framework/cfgparser/parse.go @@ -0,0 +1,391 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package config provides set of utilities for configuration parsing. +package parser + +import ( + "errors" + "fmt" + "io" + "strings" + "unicode" + + "github.com/foxcpp/maddy/framework/config/lexer" +) + +// Node struct describes a parsed configurtion block or a simple directive. +// +// name arg0 arg1 { +// children0 +// children1 +// } +type Node struct { + // Name is the first string at node's line. + Name string + // Args are any strings placed after the node name. + Args []string + + // Children slice contains all children blocks if node is a block. Can be nil. + Children []Node + + // Snippet indicates whether current parsed node is a snippet. Always false + // for all nodes returned from Read because snippets are expanded before it + // returns. + Snippet bool + + // Macro indicates whether current parsed node is a macro. Always false + // for all nodes returned from Read because macros are expanded before it + // returns. + Macro bool + + // File is the name of node's source file. + File string + + // Line is the line number where the directive is located in the source file. For + // blocks this is the line where "block header" (name + args) resides. + Line int +} + +type parseContext struct { + lexer.Dispenser + nesting int + snippets map[string][]Node + macros map[string][]string + + fileLocation string +} + +func validateNodeName(s string) error { + if len(s) == 0 { + return errors.New("empty directive name") + } + + if unicode.IsDigit([]rune(s)[0]) { + return errors.New("directive name cannot start with a digit") + } + + allowedPunct := map[rune]bool{'.': true, '-': true, '_': true} + + for _, ch := range s { + if !unicode.IsLetter(ch) && + !unicode.IsDigit(ch) && + !allowedPunct[ch] { + return errors.New("character not allowed in directive name: " + string(ch)) + } + } + + return nil +} + +// readNode reads node starting at current token pointed by the lexer's +// cursor (it should point to node name). +// +// After readNode returns, the lexer's cursor will point to the last token of the parsed +// Node. This ensures predictable cursor location independently of the EOF state. +// Thus code reading multiple nodes should call readNode then manually +// advance lexer cursor (ctx.Next) and either call readNode again or stop +// because cursor hit EOF. +// +// readNode calls readNodes if currently parsed node is a block. +func (ctx *parseContext) readNode() (Node, error) { + node := Node{} + node.File = ctx.File() + node.Line = ctx.Line() + + if ctx.Val() == "{" { + return node, ctx.SyntaxErr("block header") + } + + node.Name = ctx.Val() + if ok, name := ctx.isSnippet(node.Name); ok { + node.Name = name + node.Snippet = true + } + + var continueOnLF bool + for { + for ctx.NextArg() || (continueOnLF && ctx.NextLine()) { + continueOnLF = false + // name arg0 arg1 { + // # ^ called when we hit this token + // c0 + // c1 + // } + if ctx.Val() == "{" { + var err error + node.Children, err = ctx.readNodes() + if err != nil { + return node, err + } + break + } + + node.Args = append(node.Args, ctx.Val()) + } + + // Continue reading the same Node if the \ was used to escape the newline. + // E.g. + // name arg0 arg1 \ + // arg2 arg3 + if len(node.Args) != 0 && node.Args[len(node.Args)-1] == `\` { + last := len(node.Args) - 1 + node.Args[last] = node.Args[last][:len(node.Args[last])-1] + if len(node.Args[last]) == 0 { + node.Args = node.Args[:last] + } + continueOnLF = true + continue + } + break + } + + macroName, macroArgs, err := ctx.parseAsMacro(&node) + if err != nil { + return node, err + } + if macroName != "" { + node.Name = macroName + node.Args = macroArgs + node.Macro = true + } + + if !node.Macro && !node.Snippet { + if err := validateNodeName(node.Name); err != nil { + return node, err + } + } + + return node, nil +} + +func NodeErr(node Node, f string, args ...interface{}) error { + if node.File == "" { + return fmt.Errorf(f, args...) + } + return fmt.Errorf("%s:%d: %s", node.File, node.Line, fmt.Sprintf(f, args...)) +} + +func (ctx *parseContext) isSnippet(name string) (bool, string) { + if strings.HasPrefix(name, "(") && strings.HasSuffix(name, ")") { + return true, name[1 : len(name)-1] + } + return false, "" +} + +func (ctx *parseContext) parseAsMacro(node *Node) (macroName string, args []string, err error) { + if !strings.HasPrefix(node.Name, "$(") { + return "", nil, nil + } + if !strings.HasSuffix(node.Name, ")") { + return "", nil, ctx.Err("macro name must end with )") + } + macroName = node.Name[2 : len(node.Name)-1] + if len(node.Args) < 2 { + return macroName, nil, ctx.Err("at least 2 arguments are required") + } + if node.Args[0] != "=" { + return macroName, nil, ctx.Err("missing = in macro declaration") + } + return macroName, node.Args[1:], nil +} + +// readNodes reads nodes from the currently parsed block. +// +// The lexer's cursor should point to the opening brace +// name arg0 arg1 { #< this one +// +// c0 +// c1 +// } +// +// To stay consistent with readNode after this function returns the lexer's cursor points +// to the last token of the black (closing brace). +func (ctx *parseContext) readNodes() ([]Node, error) { + // It is not 'var res []Node' because we want empty + // but non-nil Children slice for empty braces. + res := []Node{} + + if ctx.nesting > 255 { + return res, ctx.Err("nesting limit reached") + } + + ctx.nesting++ + + var requireNewLine bool + // This loop iterates over logical lines. + // Here are some examples, '#' is placed before token where cursor is when + // another iteration of this loop starts. + // + // #a + // #a b + // #a b { + // #ac aa + // #} + // #aa bbb bbb \ + // ccc ccc + // #a b { #ac aa } + // + // As can be seen by the latest example, sometimes such logical line might + // not be terminated by an actual LF character and so this needs to be + // handled carefully. + // + // Note that if the '}' is on the same physical line, it is currently + // included as the part of the logical line, that is: + // #a b { #ac aa } + // ^------- that's the logical line + // #c d + // ^--- that's the next logical line + // This is handled by the "edge case" branch inside the loop. + for { + if requireNewLine { + if !ctx.NextLine() { + // If we can't advance cursor even without Line constraint - + // that's EOF. + if !ctx.Next() { + return res, nil + } + return res, ctx.Err("newline is required after closing brace") + } + } else if !ctx.Next() { + break + } + + // name arg0 arg1 { + // c0 + // c1 + // } + // ^ called when we hit } on separate line, + // This means block we hit end of our block. + if ctx.Val() == "}" { + ctx.nesting-- + // name arg0 arg1 { #<1 + // } } + // ^2 ^3 + // + // After #1 ctx.nesting is incremented by ctx.nesting++ before this loop. + // Then we advance cursor and hit }, we exit loop, ctx.nesting now becomes 0. + // But then the parent block reader does the same when it hits #3 - + // ctx.nesting becomes -1 and it fails. + if ctx.nesting < 0 { + return res, ctx.Err("unexpected }") + } + break + } + node, err := ctx.readNode() + if err != nil { + return res, err + } + requireNewLine = true + + shouldStop := false + + // name arg0 arg1 { + // c1 c2 } + // ^ + // Edge case, here we check if the last argument of the last node is a } + // If it is - we stop as we hit the end of our block. + if len(node.Args) != 0 && node.Args[len(node.Args)-1] == "}" { + ctx.nesting-- + if ctx.nesting < 0 { + return res, ctx.Err("unexpected }") + } + node.Args = node.Args[:len(node.Args)-1] + shouldStop = true + } + + if node.Macro { + if ctx.nesting != 0 { + return res, ctx.Err("macro declarations are only allowed at top-level") + } + + // Macro declaration itself can contain macro references. + if err := ctx.expandMacros(&node); err != nil { + return res, err + } + + // = sign is removed by parseAsMacro. + // It also cuts $( and ) from name. + ctx.macros[node.Name] = node.Args + continue + } + if node.Snippet { + if ctx.nesting != 0 { + return res, ctx.Err("snippet declarations are only allowed at top-level") + } + if len(node.Args) != 0 { + return res, ctx.Err("snippet declarations can't have arguments") + } + + ctx.snippets[node.Name] = node.Children + continue + } + + if err := ctx.expandMacros(&node); err != nil { + return res, err + } + + res = append(res, node) + if shouldStop { + break + } + } + return res, nil +} + +func readTree(r io.Reader, location string, expansionDepth int) (nodes []Node, snips map[string][]Node, macros map[string][]string, err error) { + ctx := parseContext{ + Dispenser: lexer.NewDispenser(location, r), + snippets: make(map[string][]Node), + macros: map[string][]string{}, + nesting: -1, + fileLocation: location, + } + + root := Node{} + root.File = location + root.Line = 1 + // Before parsing starts the lexer's cursor points to the non-existent + // token before the first one. From readNodes viewpoint this is opening + // brace so we don't break any requirements here. + // + // For the same reason we use -1 as a starting nesting. So readNodes + // will see this as it is reading block at nesting level 0. + root.Children, err = ctx.readNodes() + if err != nil { + return root.Children, ctx.snippets, ctx.macros, err + } + + // There is no need to check ctx.nesting < 0 because it is checked by readNodes. + if ctx.nesting > 0 { + return root.Children, ctx.snippets, ctx.macros, ctx.Err("unexpected EOF when looking for }") + } + + root, err = ctx.expandImports(root, expansionDepth) + if err != nil { + return root.Children, ctx.snippets, ctx.macros, err + } + + return root.Children, ctx.snippets, ctx.macros, nil +} + +func Read(r io.Reader, location string) (nodes []Node, err error) { + nodes, _, _, err = readTree(r, location, 0) + nodes = expandEnvironment(nodes) + return +} diff --git a/framework/cfgparser/parse_test.go b/framework/cfgparser/parse_test.go new file mode 100644 index 0000000..32929b9 --- /dev/null +++ b/framework/cfgparser/parse_test.go @@ -0,0 +1,622 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package parser + +import ( + "os" + "reflect" + "strings" + "testing" +) + +var cases = []struct { + name string + cfg string + tree []Node + fail bool +}{ + { + "single directive without args", + `a`, + []Node{ + { + Name: "a", + Args: []string{}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "single directive with args", + `a a1 a2`, + []Node{ + { + Name: "a", + Args: []string{"a1", "a2"}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "single directive with empty braces", + `a { }`, + []Node{ + { + Name: "a", + Args: []string{}, + Children: []Node{}, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "single directive with arguments and empty braces", + `a a1 a2 { }`, + []Node{ + { + Name: "a", + Args: []string{"a1", "a2"}, + Children: []Node{}, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "single directive with a block", + `a a1 a2 { + a_child1 c1arg1 c1arg2 + a_child2 c2arg1 c2arg2 + }`, + []Node{ + { + Name: "a", + Args: []string{"a1", "a2"}, + Children: []Node{ + { + Name: "a_child1", + Args: []string{"c1arg1", "c1arg2"}, + Children: nil, + File: "test", + Line: 2, + }, + { + Name: "a_child2", + Args: []string{"c2arg1", "c2arg2"}, + Children: nil, + File: "test", + Line: 3, + }, + }, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "single directive with missing closing brace", + `a {`, + nil, + true, + }, + { + "single directive with missing opening brace", + `a }`, + nil, + true, + }, + { + "two directives", + `a + b`, + []Node{ + { + Name: "a", + Args: []string{}, + Children: nil, + File: "test", + Line: 1, + }, + { + Name: "b", + Args: []string{}, + Children: nil, + File: "test", + Line: 2, + }, + }, + false, + }, + { + "two directives with arguments", + `a a1 a2 + b b1 b2`, + []Node{ + { + Name: "a", + Args: []string{"a1", "a2"}, + Children: nil, + File: "test", + Line: 1, + }, + { + Name: "b", + Args: []string{"b1", "b2"}, + Children: nil, + File: "test", + Line: 2, + }, + }, + false, + }, + { + "backslash on the end of line", + `a a1 a2 \ + a3 a4`, + []Node{ + { + Name: "a", + Args: []string{"a1", "a2", "a3", "a4"}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "directive with missing closing brace on different line", + `a a1 a2 { + a_child1 c1arg1 c1arg2 + `, + nil, + true, + }, + { + "single directive with closing brace on children's line", + `a a1 a2 { + a_child1 c1arg1 c1arg2 + a_child2 c2arg1 c2arg2 } + b`, + []Node{ + { + Name: "a", + Args: []string{"a1", "a2"}, + Children: []Node{ + { + Name: "a_child1", + Args: []string{"c1arg1", "c1arg2"}, + Children: nil, + File: "test", + Line: 2, + }, + { + Name: "a_child2", + Args: []string{"c2arg1", "c2arg2"}, + Children: nil, + File: "test", + Line: 3, + }, + }, + File: "test", + Line: 1, + }, + { + Name: "b", + Args: []string{}, + Children: nil, + File: "test", + Line: 4, + }, + }, + false, + }, + { + "single directive with childrens on the same line", + `a a1 a2 { a_child1 c1arg1 c1arg2 }`, + []Node{ + { + Name: "a", + Args: []string{"a1", "a2"}, + Children: []Node{ + { + Name: "a_child1", + Args: []string{"c1arg1", "c1arg2"}, + Children: nil, + File: "test", + Line: 1, + }, + }, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "invalid directive name", + `a-a4@%8 whatever`, + nil, + true, + }, + { + "directive name starts with a digit", + `1w whatever`, + nil, + true, + }, + { + "missing block header", + `{ a_child1 c1arg1 c1arg2 }`, + nil, + true, + }, + { + "extra closing brace", + `a { + child1 + } } + `, + nil, + true, + }, + { + "extra opening brace", + `a { { + }`, + nil, + true, + }, + { + "closing brace in next block header", + `a { + } b b1`, + nil, + true, + }, + { + "environment variable expansion", + `a {env:TESTING_VARIABLE}`, + []Node{ + { + Name: "a", + Args: []string{"ABCDEF"}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "missing environment variable expansion (unix-like syntax)", + `a {env:TESTING_VARIABLE3}`, + []Node{ + { + Name: "a", + Args: []string{""}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "incomplete environment variable syntax", + `a {env:TESTING_VARIABLE`, + []Node{ + { + Name: "a", + Args: []string{"{env:TESTING_VARIABLE"}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "snippet expansion", + `(foo) { a } + import foo`, + []Node{ + { + Name: "a", + Args: []string{}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "snippet expansion inside a block", + `(foo) { a } + foo { + boo + import foo + }`, + []Node{ + { + Name: "foo", + Args: []string{}, + Children: []Node{ + { + Name: "boo", + Args: []string{}, + File: "test", + Line: 3, + }, + { + Name: "a", + Args: []string{}, + File: "test", + Line: 1, + }, + }, + File: "test", + Line: 2, + }, + }, + false, + }, + { + "missing snippet", + `import foo`, + nil, + true, + }, + { + "unlimited recursive snippet expansion", + `(foo) { import foo } + import foo`, + nil, + true, + }, + { + "snippet declaration with args", + `(foo) a b c { }`, + nil, + true, + }, + { + "snippet declaration inside block", + `abc { + (foo) { } + }`, + nil, + true, + }, + { + "block nesting limit", + `a ` + strings.Repeat("a { ", 1000) + strings.Repeat(" }", 1000), + nil, + true, + }, + { + "macro expansion, single argument", + `$(foo) = bar + dir $(foo)`, + []Node{ + { + Name: "dir", + Args: []string{"bar"}, + Children: nil, + File: "test", + Line: 2, + }, + }, + false, + }, + { + "macro expansion, inside argument", + `$(foo) = bar + dir aaa/$(foo)/bbb`, + []Node{ + { + Name: "dir", + Args: []string{"aaa/bar/bbb"}, + Children: nil, + File: "test", + Line: 2, + }, + }, + false, + }, + { + "macro expansion, inside argument, multi-value", + `$(foo) = bar baz + dir aaa/$(foo)/bbb`, + nil, + true, + }, + { + "macro expansion, multiple arguments", + `$(foo) = bar baz + dir $(foo)`, + []Node{ + { + Name: "dir", + Args: []string{"bar", "baz"}, + Children: nil, + File: "test", + Line: 2, + }, + }, + false, + }, + { + "macro expansion, undefined", + `dir $(foo)`, + []Node{ + { + Name: "dir", + Args: []string{}, + Children: nil, + File: "test", + Line: 1, + }, + }, + false, + }, + { + "macro expansion, empty", + `$(foo) =`, + nil, + true, + }, + { + "macro expansion, name replacement", + `$(foo) = a b + $(foo) 1`, + nil, + true, + }, + { + "macro expansion, missing =", + `$(foo) a b + $(foo) 1`, + nil, + true, + }, + { + "macro expansion, not on top level", + `a { + $(foo) = a b + } + $(foo) 1`, + nil, + true, + }, + { + "macro expansion, nested", + `$(foo) = a + $(bar) = $(foo) b + dir $(bar)`, + []Node{ + { + Name: "dir", + Args: []string{"a", "b"}, + Children: nil, + File: "test", + Line: 3, + }, + }, + false, + }, + { + "macro expansion, used inside snippet", + `$(foo) = a + (bar) { + dir $(foo) + } + import bar`, + []Node{ + { + Name: "dir", + Args: []string{"a"}, + Children: nil, + File: "test", + Line: 3, + }, + }, + false, + }, + { + "macro expansion, used inside snippet, defined after", + ` + (bar) { + dir $(foo) + } + $(foo) = a + import bar`, + []Node{ + { + Name: "dir", + Args: []string{}, + Children: nil, + File: "test", + Line: 3, + }, + }, + false, + }, +} + +func printTree(t *testing.T, root Node, indent int) { + t.Log(strings.Repeat(" ", indent)+root.Name, root.Args) + for _, child := range root.Children { + t.Log(child, indent+1) + } +} + +func TestRead(t *testing.T) { + os.Setenv("TESTING_VARIABLE", "ABCDEF") + os.Setenv("TESTING_VARIABLE2", "ABC2 DEF2") + + for _, case_ := range cases { + t.Run(case_.name, func(t *testing.T) { + tree, err := Read(strings.NewReader(case_.cfg), "test") + if !case_.fail && err != nil { + t.Error("unexpected failure:", err) + return + } + if case_.fail { + if err == nil { + t.Log("expected failure but Read succeeded") + t.Log("got tree:") + t.Logf("%+v", tree) + for _, node := range tree { + printTree(t, node, 0) + } + t.Fail() + return + } + return + } + + if !reflect.DeepEqual(case_.tree, tree) { + t.Log("parse result mismatch") + t.Log("expected:") + t.Logf("%+#v", case_.tree) + for _, node := range case_.tree { + printTree(t, node, 0) + } + t.Log("actual:") + t.Logf("%+#v", tree) + for _, node := range tree { + printTree(t, node, 0) + } + t.Fail() + } + }) + } +} diff --git a/framework/config/config.go b/framework/config/config.go new file mode 100644 index 0000000..3dd9911 --- /dev/null +++ b/framework/config/config.go @@ -0,0 +1,36 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package config + +import ( + "fmt" + + parser "github.com/foxcpp/maddy/framework/cfgparser" +) + +type ( + Node = parser.Node +) + +func NodeErr(node Node, f string, args ...interface{}) error { + if node.File == "" { + return fmt.Errorf(f, args...) + } + return fmt.Errorf("%s:%d: %s", node.File, node.Line, fmt.Sprintf(f, args...)) +} diff --git a/framework/config/directories.go b/framework/config/directories.go new file mode 100644 index 0000000..013dfc9 --- /dev/null +++ b/framework/config/directories.go @@ -0,0 +1,47 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package config + +var ( + // StateDirectory contains the path to the directory that + // should be used to store any data that should be + // preserved between sessions. + // + // Value of this variable must not change after initialization + // in cmd/maddy/main.go. + StateDirectory string + + // RuntimeDirectory contains the path to the directory that + // should be used to store any temporary data. + // + // It should be preferred over os.TempDir, which is + // global and world-readable on most systems, while + // RuntimeDirectory can be dedicated for maddy. + // + // Value of this variable must not change after initialization + // in cmd/maddy/main.go. + RuntimeDirectory string + + // LibexecDirectory contains the path to the directory + // where helper binaries should be searched. + // + // Value of this variable must not change after initialization + // in cmd/maddy/main.go. + LibexecDirectory string +) diff --git a/framework/config/endpoint.go b/framework/config/endpoint.go new file mode 100644 index 0000000..210d57b --- /dev/null +++ b/framework/config/endpoint.go @@ -0,0 +1,142 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package config + +import ( + "fmt" + "net" + "net/url" + "path/filepath" + "strings" +) + +// Endpoint represents a site address. It contains the original input value, +// and the component parts of an address. The component parts may be updated to +// the correct values as setup proceeds, but the original value should never be +// changed. +type Endpoint struct { + Original, Scheme, Host, Port, Path string +} + +// String returns a human-friendly print of the address. +func (e Endpoint) String() string { + if e.Original != "" { + return e.Original + } + + if e.Scheme == "unix" { + return "unix://" + e.Path + } + + if e.Host == "" && e.Port == "" { + return "" + } + s := e.Scheme + if s != "" { + s += "://" + } + + host := e.Host + if strings.Contains(host, ":") { + host = "[" + host + "]" + } + s += host + + if e.Port != "" { + s += ":" + e.Port + } + if e.Path != "" { + s += e.Path + } + return s +} + +func (e Endpoint) Network() string { + if e.Scheme == "unix" { + return "unix" + } + return "tcp" +} + +func (e Endpoint) Address() string { + if e.Scheme == "unix" { + return e.Path + } + return net.JoinHostPort(e.Host, e.Port) +} + +func (e Endpoint) IsTLS() bool { + return e.Scheme == "tls" +} + +// ParseEndpoint parses an endpoint string into a structured format with separate +// scheme, host, port, and path portions, as well as the original input string. +func ParseEndpoint(str string) (Endpoint, error) { + input := str + + u, err := url.Parse(str) + if err != nil { + return Endpoint{}, err + } + + switch u.Scheme { + case "tcp", "tls": + // ALL GREEN + + // scheme:OPAQUE URL syntax + if u.Host == "" && u.Opaque != "" { + u.Host = u.Opaque + } + case "unix": + // scheme:OPAQUE URL syntax + if u.Path == "" && u.Opaque != "" { + u.Path = u.Opaque + } + + var actualPath string + if u.Host != "" { + actualPath += u.Host + } + if u.Path != "" { + actualPath += u.Path + } + + if !filepath.IsAbs(actualPath) { + actualPath = filepath.Join(RuntimeDirectory, actualPath) + } + + return Endpoint{Original: input, Scheme: u.Scheme, Path: actualPath}, err + default: + return Endpoint{}, fmt.Errorf("unsupported scheme: %s (%+v)", input, u) + } + + // separate host and port + host, port, err := net.SplitHostPort(u.Host) + if err != nil { + host, port, err = net.SplitHostPort(u.Host + ":") + if err != nil { + host = u.Host + } + } + if port == "" { + return Endpoint{}, fmt.Errorf("port is required") + } + + return Endpoint{Original: input, Scheme: u.Scheme, Host: host, Port: port, Path: u.Path}, err +} diff --git a/framework/config/endpoint_test.go b/framework/config/endpoint_test.go new file mode 100644 index 0000000..b786ef0 --- /dev/null +++ b/framework/config/endpoint_test.go @@ -0,0 +1,55 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package config + +import ( + "reflect" + "testing" +) + +func TestStandardizeAddress(t *testing.T) { + for _, expected := range []Endpoint{ + {Original: "tcp://0.0.0.0:10025", Scheme: "tcp", Host: "0.0.0.0", Port: "10025"}, + {Original: "tcp://[::]:10025", Scheme: "tcp", Host: "::", Port: "10025"}, + {Original: "tcp:127.0.0.1:10025", Scheme: "tcp", Host: "127.0.0.1", Port: "10025"}, + {Original: "unix://path", Scheme: "unix", Host: "", Path: "path", Port: ""}, + {Original: "unix:path", Scheme: "unix", Host: "", Path: "path", Port: ""}, + {Original: "unix:/path", Scheme: "unix", Host: "", Path: "/path", Port: ""}, + {Original: "unix:///path", Scheme: "unix", Host: "", Path: "/path", Port: ""}, + {Original: "unix://also/path", Scheme: "unix", Host: "", Path: "also/path", Port: ""}, + {Original: "unix:///also/path", Scheme: "unix", Host: "", Path: "/also/path", Port: ""}, + {Original: "tls://0.0.0.0:10025", Scheme: "tls", Host: "0.0.0.0", Port: "10025"}, + {Original: "tls:0.0.0.0:10025", Scheme: "tls", Host: "0.0.0.0", Port: "10025"}, + } { + actual, err := ParseEndpoint(expected.Original) + if err != nil { + t.Errorf("Unexpected failure for %s: %v", expected.Original, err) + return + } + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Didn't parse URL %q correctly\ngot %#v\nwant %#v", expected.Original, actual, expected) + continue + } + + if actual.String() != expected.Original { + t.Errorf("actual.String() = %s, want %s", actual.String(), expected.Original) + } + } +} diff --git a/framework/config/lexer/LICENSE.APACHE b/framework/config/lexer/LICENSE.APACHE new file mode 100644 index 0000000..8dada3e --- /dev/null +++ b/framework/config/lexer/LICENSE.APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/framework/config/lexer/README.md b/framework/config/lexer/README.md new file mode 100644 index 0000000..554a7a2 --- /dev/null +++ b/framework/config/lexer/README.md @@ -0,0 +1,20 @@ +caddyfile lexer copied from [caddy](https://github.com/caddyserver/caddy) project. + +Taken from the following commit: +``` +commit ed4c2775e46b924d4851e04cc281633b1b2c15af +Author: Alexander Danilov +Date: Wed Aug 21 20:13:34 2019 +0300 + + main: log caddy version on start (#2717) + +``` + +License of the original code is included in LICENSE.APACHE file in this +directory. + +No signficant changes was made to the code (e.g. it is safe to update it from +caddy repo). + +The code is copied because caddy brings quite a lot of dependencies we don't +use and this slows down many tools. diff --git a/framework/config/lexer/dispenser.go b/framework/config/lexer/dispenser.go new file mode 100644 index 0000000..db710f1 --- /dev/null +++ b/framework/config/lexer/dispenser.go @@ -0,0 +1,264 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Copyright 2015 Light Code Labs, LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lexer + +import ( + "errors" + "fmt" + "io" + "strings" +) + +// Dispenser is a type that dispenses tokens, similarly to a lexer, +// except that it can do so with some notion of structure and has +// some really convenient methods. +type Dispenser struct { + filename string + tokens []Token + cursor int + nesting int +} + +// NewDispenser returns a Dispenser, ready to use for parsing the given input. +func NewDispenser(filename string, input io.Reader) Dispenser { + tokens, _ := allTokens(input) // ignoring error because nothing to do with it + return Dispenser{ + filename: filename, + tokens: tokens, + cursor: -1, + } +} + +// NewDispenserTokens returns a Dispenser filled with the given tokens. +func NewDispenserTokens(filename string, tokens []Token) Dispenser { + return Dispenser{ + filename: filename, + tokens: tokens, + cursor: -1, + } +} + +// Next loads the next token. Returns true if a token +// was loaded; false otherwise. If false, all tokens +// have been consumed. +func (d *Dispenser) Next() bool { + if d.cursor < len(d.tokens)-1 { + d.cursor++ + return true + } + return false +} + +// NextArg loads the next token if it is on the same +// line. Returns true if a token was loaded; false +// otherwise. If false, all tokens on the line have +// been consumed. It handles imported tokens correctly. +func (d *Dispenser) NextArg() bool { + if d.cursor < 0 { + d.cursor++ + return true + } + if d.cursor >= len(d.tokens) { + return false + } + if d.cursor < len(d.tokens)-1 && + d.tokens[d.cursor].File == d.tokens[d.cursor+1].File && + d.tokens[d.cursor].Line+d.numLineBreaks(d.cursor) == d.tokens[d.cursor+1].Line { + d.cursor++ + return true + } + return false +} + +// NextLine loads the next token only if it is not on the same +// line as the current token, and returns true if a token was +// loaded; false otherwise. If false, there is not another token +// or it is on the same line. It handles imported tokens correctly. +func (d *Dispenser) NextLine() bool { + if d.cursor < 0 { + d.cursor++ + return true + } + if d.cursor >= len(d.tokens) { + return false + } + if d.cursor < len(d.tokens)-1 && + (d.tokens[d.cursor].File != d.tokens[d.cursor+1].File || + d.tokens[d.cursor].Line+d.numLineBreaks(d.cursor) < d.tokens[d.cursor+1].Line) { + d.cursor++ + return true + } + return false +} + +// NextBlock can be used as the condition of a for loop +// to load the next token as long as it opens a block or +// is already in a block. It returns true if a token was +// loaded, or false when the block's closing curly brace +// was loaded and thus the block ended. Nested blocks are +// not supported. +func (d *Dispenser) NextBlock() bool { + if d.nesting > 0 { + d.Next() + if d.Val() == "}" { + d.nesting-- + return false + } + return true + } + if !d.NextArg() { // block must open on same line + return false + } + if d.Val() != "{" { + d.cursor-- // roll back if not opening brace + return false + } + d.Next() + if d.Val() == "}" { + // Open and then closed right away + return false + } + d.nesting++ + return true +} + +// Val gets the text of the current token. If there is no token +// loaded, it returns empty string. +func (d *Dispenser) Val() string { + if d.cursor < 0 || d.cursor >= len(d.tokens) { + return "" + } + return d.tokens[d.cursor].Text +} + +// Line gets the line number of the current token. If there is no token +// loaded, it returns 0. +func (d *Dispenser) Line() int { + if d.cursor < 0 || d.cursor >= len(d.tokens) { + return 0 + } + return d.tokens[d.cursor].Line +} + +// File gets the filename of the current token. If there is no token loaded, +// it returns the filename originally given when parsing started. +func (d *Dispenser) File() string { + if d.cursor < 0 || d.cursor >= len(d.tokens) { + return d.filename + } + if tokenFilename := d.tokens[d.cursor].File; tokenFilename != "" { + return tokenFilename + } + return d.filename +} + +// Args is a convenience function that loads the next arguments +// (tokens on the same line) into an arbitrary number of strings +// pointed to in targets. If there are fewer tokens available +// than string pointers, the remaining strings will not be changed +// and false will be returned. If there were enough tokens available +// to fill the arguments, then true will be returned. +func (d *Dispenser) Args(targets ...*string) bool { + enough := true + for i := 0; i < len(targets); i++ { + if !d.NextArg() { + enough = false + break + } + *targets[i] = d.Val() + } + return enough +} + +// RemainingArgs loads any more arguments (tokens on the same line) +// into a slice and returns them. Open curly brace tokens also indicate +// the end of arguments, and the curly brace is not included in +// the return value nor is it loaded. +func (d *Dispenser) RemainingArgs() []string { + var args []string + + for d.NextArg() { + if d.Val() == "{" { + d.cursor-- + break + } + args = append(args, d.Val()) + } + + return args +} + +// ArgErr returns an argument error, meaning that another +// argument was expected but not found. In other words, +// a line break or open curly brace was encountered instead of +// an argument. +func (d *Dispenser) ArgErr() error { + if d.Val() == "{" { + return d.Err("Unexpected token '{', expecting argument") + } + return d.Errf("Wrong argument count or unexpected line ending after '%s'", d.Val()) +} + +// SyntaxErr creates a generic syntax error which explains what was +// found and what was expected. +func (d *Dispenser) SyntaxErr(expected string) error { + msg := fmt.Sprintf("%s:%d - Syntax error: Unexpected token '%s', expecting '%s'", d.File(), d.Line(), d.Val(), expected) + return errors.New(msg) +} + +// EOFErr returns an error indicating that the dispenser reached +// the end of the input when searching for the next token. +func (d *Dispenser) EOFErr() error { + return d.Errf("Unexpected EOF") +} + +// Err generates a custom parse-time error with a message of msg. +func (d *Dispenser) Err(msg string) error { + msg = fmt.Sprintf("%s:%d - Error during parsing: %s", d.File(), d.Line(), msg) + return errors.New(msg) +} + +// Errf is like Err, but for formatted error messages +func (d *Dispenser) Errf(format string, args ...interface{}) error { + return d.Err(fmt.Sprintf(format, args...)) +} + +// numLineBreaks counts how many line breaks are in the token +// value given by the token index tknIdx. It returns 0 if the +// token does not exist or there are no line breaks. +func (d *Dispenser) numLineBreaks(tknIdx int) int { + if tknIdx < 0 || tknIdx >= len(d.tokens) { + return 0 + } + return strings.Count(d.tokens[tknIdx].Text, "\n") +} diff --git a/framework/config/lexer/dispenser_test.go b/framework/config/lexer/dispenser_test.go new file mode 100644 index 0000000..1f75bb1 --- /dev/null +++ b/framework/config/lexer/dispenser_test.go @@ -0,0 +1,324 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Copyright 2015 Light Code Labs, LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lexer + +import ( + "reflect" + "strings" + "testing" +) + +func TestDispenser_Val_Next(t *testing.T) { + input := `host:port + dir1 arg1 + dir2 arg2 arg3 + dir3` + d := NewDispenser("Testfile", strings.NewReader(input)) + + if val := d.Val(); val != "" { + t.Fatalf("Val(): Should return empty string when no token loaded; got '%s'", val) + } + + assertNext := func(shouldLoad bool, expectedCursor int, expectedVal string) { + if loaded := d.Next(); loaded != shouldLoad { + t.Errorf("Next(): Expected %v but got %v instead (val '%s')", shouldLoad, loaded, d.Val()) + } + if d.cursor != expectedCursor { + t.Errorf("Expected cursor to be %d, but was %d", expectedCursor, d.cursor) + } + if d.nesting != 0 { + t.Errorf("Nesting should be 0, was %d instead", d.nesting) + } + if val := d.Val(); val != expectedVal { + t.Errorf("Val(): Expected '%s' but got '%s'", expectedVal, val) + } + } + + assertNext(true, 0, "host:port") + assertNext(true, 1, "dir1") + assertNext(true, 2, "arg1") + assertNext(true, 3, "dir2") + assertNext(true, 4, "arg2") + assertNext(true, 5, "arg3") + assertNext(true, 6, "dir3") + // Note: This next test simply asserts existing behavior. + // If desired, we may wish to empty the token value after + // reading past the EOF. Open an issue if you want this change. + assertNext(false, 6, "dir3") +} + +func TestDispenser_NextArg(t *testing.T) { + input := `dir1 arg1 + dir2 arg2 arg3 + dir3` + d := NewDispenser("Testfile", strings.NewReader(input)) + + assertNext := func(shouldLoad bool, expectedVal string, expectedCursor int) { + if d.Next() != shouldLoad { + t.Errorf("Next(): Should load token but got false instead (val: '%s')", d.Val()) + } + if d.cursor != expectedCursor { + t.Errorf("Next(): Expected cursor to be at %d, but it was %d", expectedCursor, d.cursor) + } + if val := d.Val(); val != expectedVal { + t.Errorf("Val(): Expected '%s' but got '%s'", expectedVal, val) + } + } + + assertNextArg := func(expectedVal string, loadAnother bool, expectedCursor int) { + if !d.NextArg() { + t.Error("NextArg(): Should load next argument but got false instead") + } + if d.cursor != expectedCursor { + t.Errorf("NextArg(): Expected cursor to be at %d, but it was %d", expectedCursor, d.cursor) + } + if val := d.Val(); val != expectedVal { + t.Errorf("Val(): Expected '%s' but got '%s'", expectedVal, val) + } + if !loadAnother { + if d.NextArg() { + t.Fatalf("NextArg(): Should NOT load another argument, but got true instead (val: '%s')", d.Val()) + } + if d.cursor != expectedCursor { + t.Errorf("NextArg(): Expected cursor to remain at %d, but it was %d", expectedCursor, d.cursor) + } + } + } + + assertNext(true, "dir1", 0) + assertNextArg("arg1", false, 1) + assertNext(true, "dir2", 2) + assertNextArg("arg2", true, 3) + assertNextArg("arg3", false, 4) + assertNext(true, "dir3", 5) + assertNext(false, "dir3", 5) +} + +func TestDispenser_NextLine(t *testing.T) { + input := `host:port + dir1 arg1 + dir2 arg2 arg3` + d := NewDispenser("Testfile", strings.NewReader(input)) + + assertNextLine := func(shouldLoad bool, expectedVal string, expectedCursor int) { + if d.NextLine() != shouldLoad { + t.Errorf("NextLine(): Should load token but got false instead (val: '%s')", d.Val()) + } + if d.cursor != expectedCursor { + t.Errorf("NextLine(): Expected cursor to be %d, instead was %d", expectedCursor, d.cursor) + } + if val := d.Val(); val != expectedVal { + t.Errorf("Val(): Expected '%s' but got '%s'", expectedVal, val) + } + } + + assertNextLine(true, "host:port", 0) + assertNextLine(true, "dir1", 1) + assertNextLine(false, "dir1", 1) + d.Next() // arg1 + assertNextLine(true, "dir2", 3) + assertNextLine(false, "dir2", 3) + d.Next() // arg2 + assertNextLine(false, "arg2", 4) + d.Next() // arg3 + assertNextLine(false, "arg3", 5) +} + +func TestDispenser_NextBlock(t *testing.T) { + input := `foobar1 { + sub1 arg1 + sub2 + } + foobar2 { + }` + d := NewDispenser("Testfile", strings.NewReader(input)) + + assertNextBlock := func(shouldLoad bool, expectedCursor, expectedNesting int) { + if loaded := d.NextBlock(); loaded != shouldLoad { + t.Errorf("NextBlock(): Should return %v but got %v", shouldLoad, loaded) + } + if d.cursor != expectedCursor { + t.Errorf("NextBlock(): Expected cursor to be %d, was %d", expectedCursor, d.cursor) + } + if d.nesting != expectedNesting { + t.Errorf("NextBlock(): Nesting should be %d, not %d", expectedNesting, d.nesting) + } + } + + assertNextBlock(false, -1, 0) + d.Next() // foobar1 + assertNextBlock(true, 2, 1) + assertNextBlock(true, 3, 1) + assertNextBlock(true, 4, 1) + assertNextBlock(false, 5, 0) + d.Next() // foobar2 + assertNextBlock(false, 8, 0) // empty block is as if it didn't exist +} + +func TestDispenser_Args(t *testing.T) { + var s1, s2, s3 string + input := `dir1 arg1 arg2 arg3 + dir2 arg4 arg5 + dir3 arg6 arg7 + dir4` + d := NewDispenser("Testfile", strings.NewReader(input)) + + d.Next() // dir1 + + // As many strings as arguments + if all := d.Args(&s1, &s2, &s3); !all { + t.Error("Args(): Expected true, got false") + } + if s1 != "arg1" { + t.Errorf("Args(): Expected s1 to be 'arg1', got '%s'", s1) + } + if s2 != "arg2" { + t.Errorf("Args(): Expected s2 to be 'arg2', got '%s'", s2) + } + if s3 != "arg3" { + t.Errorf("Args(): Expected s3 to be 'arg3', got '%s'", s3) + } + + d.Next() // dir2 + + // More strings than arguments + if all := d.Args(&s1, &s2, &s3); all { + t.Error("Args(): Expected false, got true") + } + if s1 != "arg4" { + t.Errorf("Args(): Expected s1 to be 'arg4', got '%s'", s1) + } + if s2 != "arg5" { + t.Errorf("Args(): Expected s2 to be 'arg5', got '%s'", s2) + } + if s3 != "arg3" { + t.Errorf("Args(): Expected s3 to be unchanged ('arg3'), instead got '%s'", s3) + } + + // (quick cursor check just for kicks and giggles) + if d.cursor != 6 { + t.Errorf("Cursor should be 6, but is %d", d.cursor) + } + + d.Next() // dir3 + + // More arguments than strings + if all := d.Args(&s1); !all { + t.Error("Args(): Expected true, got false") + } + if s1 != "arg6" { + t.Errorf("Args(): Expected s1 to be 'arg6', got '%s'", s1) + } + + d.Next() // dir4 + + // No arguments or strings + if all := d.Args(); !all { + t.Error("Args(): Expected true, got false") + } + + // No arguments but at least one string + if all := d.Args(&s1); all { + t.Error("Args(): Expected false, got true") + } +} + +func TestDispenser_RemainingArgs(t *testing.T) { + input := `dir1 arg1 arg2 arg3 + dir2 arg4 arg5 + dir3 arg6 { arg7 + dir4` + d := NewDispenser("Testfile", strings.NewReader(input)) + + d.Next() // dir1 + + args := d.RemainingArgs() + if expected := []string{"arg1", "arg2", "arg3"}; !reflect.DeepEqual(args, expected) { + t.Errorf("RemainingArgs(): Expected %v, got %v", expected, args) + } + + d.Next() // dir2 + + args = d.RemainingArgs() + if expected := []string{"arg4", "arg5"}; !reflect.DeepEqual(args, expected) { + t.Errorf("RemainingArgs(): Expected %v, got %v", expected, args) + } + + d.Next() // dir3 + + args = d.RemainingArgs() + if expected := []string{"arg6"}; !reflect.DeepEqual(args, expected) { + t.Errorf("RemainingArgs(): Expected %v, got %v", expected, args) + } + + d.Next() // { + d.Next() // arg7 + d.Next() // dir4 + + args = d.RemainingArgs() + if len(args) != 0 { + t.Errorf("RemainingArgs(): Expected %v, got %v", []string{}, args) + } +} + +func TestDispenser_ArgErr_Err(t *testing.T) { + input := `dir1 { + } + dir2 arg1 arg2` + d := NewDispenser("Testfile", strings.NewReader(input)) + + d.cursor = 1 // { + + if err := d.ArgErr(); err == nil || !strings.Contains(err.Error(), "{") { + t.Errorf("ArgErr(): Expected an error message with { in it, but got '%v'", err) + } + + d.cursor = 5 // arg2 + + if err := d.ArgErr(); err == nil || !strings.Contains(err.Error(), "arg2") { + t.Errorf("ArgErr(): Expected an error message with 'arg2' in it; got '%v'", err) + } + + err := d.Err("foobar") + if err == nil { + t.Fatalf("Err(): Expected an error, got nil") + } + + if !strings.Contains(err.Error(), "Testfile:3") { + t.Errorf("Expected error message with filename:line in it; got '%v'", err) + } + + if !strings.Contains(err.Error(), "foobar") { + t.Errorf("Expected error message with custom message in it ('foobar'); got '%v'", err) + } +} diff --git a/framework/config/lexer/lexer.go b/framework/config/lexer/lexer.go new file mode 100644 index 0000000..38952df --- /dev/null +++ b/framework/config/lexer/lexer.go @@ -0,0 +1,178 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Copyright 2015 Light Code Labs, LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lexer + +import ( + "bufio" + "io" + "unicode" +) + +type ( + // lexer is a utility which can get values, token by + // token, from a Reader. A token is a word, and tokens + // are separated by whitespace. A word can be enclosed + // in quotes if it contains whitespace. + lexer struct { + reader *bufio.Reader + token Token + line int + + lastErr error + } + + // Token represents a single parsable unit. + Token struct { + File string + Line int + Text string + } +) + +// load prepares the lexer to scan an input for tokens. +// It discards any leading byte order mark. +func (l *lexer) load(input io.Reader) error { + l.reader = bufio.NewReader(input) + l.line = 1 + + // discard byte order mark, if present + firstCh, _, err := l.reader.ReadRune() + if err != nil { + return err + } + if firstCh != 0xFEFF { + err := l.reader.UnreadRune() + if err != nil { + return err + } + } + + return nil +} + +func (l *lexer) err() error { + return l.lastErr +} + +// next loads the next token into the lexer. +// +// A token is delimited by whitespace, unless the token starts with a quotes +// character (") in which case the token goes until the closing quotes (the +// enclosing quotes are not included). Inside quoted strings, quotes may be +// escaped with a preceding \ character. No other chars may be escaped. Curly +// braces ('{', '}') are emitted as a separate tokens. +// +// The rest of the line is skipped if a "#" character is read in. +// +// Returns true if a token was loaded; false otherwise. If read from +// underlying Reader fails, next() returns false and err() will return the +// error occurred. +func (l *lexer) next() bool { + var val []rune + var comment, quoted, escaped bool + + makeToken := func() bool { + l.token.Text = string(val) + l.lastErr = nil + return true + } + + for { + ch, _, err := l.reader.ReadRune() + if err != nil { + if len(val) > 0 { + return makeToken() + } + if err == io.EOF { + return false + } + l.lastErr = err + return false + } + + if quoted { + if !escaped { + if ch == '\\' { + escaped = true + continue + } else if ch == '"' { + return makeToken() + } + } + if ch == '\n' { + l.line++ + } + if escaped { + // only escape quotes + if ch != '"' { + val = append(val, '\\') + } + } + val = append(val, ch) + escaped = false + continue + } + + if unicode.IsSpace(ch) { + if ch == '\r' { + continue + } + if ch == '\n' { + l.line++ + comment = false + } + if len(val) > 0 { + return makeToken() + } + continue + } + + if ch == '#' { + comment = true + } + + if comment { + continue + } + + if len(val) == 0 { + l.token = Token{Line: l.line} + if ch == '"' { + quoted = true + continue + } + } + + val = append(val, ch) + } +} diff --git a/framework/config/lexer/lexer_test.go b/framework/config/lexer/lexer_test.go new file mode 100644 index 0000000..5f42f2f --- /dev/null +++ b/framework/config/lexer/lexer_test.go @@ -0,0 +1,206 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Copyright 2015 Light Code Labs, LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lexer + +import ( + "log" + "strings" + "testing" +) + +type lexerTestCase struct { + input string + expected []Token +} + +func TestLexer(t *testing.T) { + testCases := []lexerTestCase{ + { + input: `host:123`, + expected: []Token{ + {Line: 1, Text: "host:123"}, + }, + }, + { + input: `host:123 + + directive`, + expected: []Token{ + {Line: 1, Text: "host:123"}, + {Line: 3, Text: "directive"}, + }, + }, + { + input: `host:123 { + directive + }`, + expected: []Token{ + {Line: 1, Text: "host:123"}, + {Line: 1, Text: "{"}, + {Line: 2, Text: "directive"}, + {Line: 3, Text: "}"}, + }, + }, + { + input: `host:123 { directive }`, + expected: []Token{ + {Line: 1, Text: "host:123"}, + {Line: 1, Text: "{"}, + {Line: 1, Text: "directive"}, + {Line: 1, Text: "}"}, + }, + }, + { + input: `host:123 { + #comment + directive + # comment + foobar # another comment + }`, + expected: []Token{ + {Line: 1, Text: "host:123"}, + {Line: 1, Text: "{"}, + {Line: 3, Text: "directive"}, + {Line: 5, Text: "foobar"}, + {Line: 6, Text: "}"}, + }, + }, + { + input: `a "quoted value" b + foobar`, + expected: []Token{ + {Line: 1, Text: "a"}, + {Line: 1, Text: "quoted value"}, + {Line: 1, Text: "b"}, + {Line: 2, Text: "foobar"}, + }, + }, + { + input: `A "quoted \"value\" inside" B`, + expected: []Token{ + {Line: 1, Text: "A"}, + {Line: 1, Text: `quoted "value" inside`}, + {Line: 1, Text: "B"}, + }, + }, + { + input: `"don't\escape"`, + expected: []Token{ + {Line: 1, Text: `don't\escape`}, + }, + }, + { + input: `"don't\\escape"`, + expected: []Token{ + {Line: 1, Text: `don't\\escape`}, + }, + }, + { + input: `A "quoted value with line + break inside" { + foobar + }`, + expected: []Token{ + {Line: 1, Text: "A"}, + {Line: 1, Text: "quoted value with line\n\t\t\t\t\tbreak inside"}, + {Line: 2, Text: "{"}, + {Line: 3, Text: "foobar"}, + {Line: 4, Text: "}"}, + }, + }, + { + input: `"C:\php\php-cgi.exe"`, + expected: []Token{ + {Line: 1, Text: `C:\php\php-cgi.exe`}, + }, + }, + { + input: `empty "" string`, + expected: []Token{ + {Line: 1, Text: `empty`}, + {Line: 1, Text: ``}, + {Line: 1, Text: `string`}, + }, + }, + { + input: "skip those\r\nCR characters", + expected: []Token{ + {Line: 1, Text: "skip"}, + {Line: 1, Text: "those"}, + {Line: 2, Text: "CR"}, + {Line: 2, Text: "characters"}, + }, + }, + { + input: "\xEF\xBB\xBF:8080", // test with leading byte order mark + expected: []Token{ + {Line: 1, Text: ":8080"}, + }, + }, + } + + for i, testCase := range testCases { + actual := tokenize(testCase.input) + lexerCompare(t, i, testCase.expected, actual) + } +} + +func tokenize(input string) (tokens []Token) { + l := lexer{} + if err := l.load(strings.NewReader(input)); err != nil { + log.Printf("[ERROR] load failed: %v", err) + } + for l.next() { + tokens = append(tokens, l.token) + } + return +} + +func lexerCompare(t *testing.T, n int, expected, actual []Token) { + if len(expected) != len(actual) { + t.Errorf("Test case %d: expected %d token(s) but got %d", n, len(expected), len(actual)) + } + + for i := 0; i < len(actual) && i < len(expected); i++ { + if actual[i].Line != expected[i].Line { + t.Errorf("Test case %d token %d ('%s'): expected line %d but was line %d", + n, i, expected[i].Text, expected[i].Line, actual[i].Line) + break + } + if actual[i].Text != expected[i].Text { + t.Errorf("Test case %d token %d: expected text '%s' but was '%s'", + n, i, expected[i].Text, actual[i].Text) + break + } + } +} diff --git a/framework/config/lexer/parse.go b/framework/config/lexer/parse.go new file mode 100644 index 0000000..34c8d79 --- /dev/null +++ b/framework/config/lexer/parse.go @@ -0,0 +1,42 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package lexer + +import ( + "io" +) + +// allTokens lexes the entire input, but does not parse it. +// It returns all the tokens from the input, unstructured +// and in order. +func allTokens(input io.Reader) ([]Token, error) { + l := new(lexer) + err := l.load(input) + if err != nil { + return nil, err + } + var tokens []Token + for l.next() { + tokens = append(tokens, l.token) + } + if err := l.err(); err != nil { + return nil, err + } + return tokens, nil +} diff --git a/framework/config/map.go b/framework/config/map.go new file mode 100644 index 0000000..c85c19f --- /dev/null +++ b/framework/config/map.go @@ -0,0 +1,743 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package config + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "time" + "unicode" +) + +type matcher struct { + name string + required bool + inheritGlobal bool + defaultVal func() (interface{}, error) + mapper func(*Map, Node) (interface{}, error) + store *reflect.Value + + customCallback func(*Map, Node) error +} + +func (m *matcher) assign(val interface{}) { + valRefl := reflect.ValueOf(val) + // Convert untyped nil into typed nil. Otherwise it will panic. + if !valRefl.IsValid() { + valRefl = reflect.Zero(m.store.Type()) + } + + m.store.Set(valRefl) +} + +// Map structure implements reflection-based conversion between configuration +// directives and Go variables. +type Map struct { + allowUnknown bool + + // All values saved by Map during processing. + Values map[string]interface{} + + entries map[string]matcher + + // Values used by Process as default values if inheritGlobal is true. + Globals map[string]interface{} + // Config block used by Process. + Block Node +} + +func NewMap(globals map[string]interface{}, block Node) *Map { + return &Map{Globals: globals, Block: block} +} + +// AllowUnknown makes config.Map skip unknown configuration directives instead +// of failing. +func (m *Map) AllowUnknown() { + m.allowUnknown = true +} + +// EnumList maps a configuration directive to a []string variable. +// +// Directive must be in form 'name string1 string2' where each string should be from *allowed* +// slice. At least one argument should be present. +// +// See Map.Custom for description of inheritGlobal and required. +func (m *Map) EnumList(name string, inheritGlobal, required bool, allowed, defaultVal []string, store *[]string) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare a block here") + } + if len(node.Args) == 0 { + return nil, NodeErr(node, "expected at least one argument") + } + + for _, arg := range node.Args { + isAllowed := false + for _, str := range allowed { + if str == arg { + isAllowed = true + } + } + if !isAllowed { + return nil, NodeErr(node, "invalid argument, valid values are: %v", allowed) + } + } + + return node.Args, nil + }, store) +} + +// Enum maps a configuration directive to a string variable. +// +// Directive must be in form 'name string' where string should be from *allowed* +// slice. That string argument will be stored in store variable. +// +// See Map.Custom for description of inheritGlobal and required. +func (m *Map) Enum(name string, inheritGlobal, required bool, allowed []string, defaultVal string, store *string) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare a block here") + } + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected exactly one argument") + } + + for _, str := range allowed { + if str == node.Args[0] { + return node.Args[0], nil + } + } + + return nil, NodeErr(node, "invalid argument, valid values are: %v", allowed) + }, store) +} + +// EnumMapped is similar to Map.Enum but maps a stirng to a custom type. +func EnumMapped[V any](m *Map, name string, inheritGlobal, required bool, mapped map[string]V, defaultVal V, store *V) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare a block here") + } + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected exactly one argument") + } + + val, ok := mapped[node.Args[0]] + if !ok { + validValues := make([]string, 0, len(mapped)) + for k := range mapped { + validValues = append(validValues, k) + } + return nil, NodeErr(node, "invalid argument, valid values are: %v", validValues) + } + + return val, nil + }, store) +} + +// EnumListMapped is similar to Map.EnumList but maps a stirng to a custom type. +func EnumListMapped[V any](m *Map, name string, inheritGlobal, required bool, mapped map[string]V, defaultVal []V, store *[]V) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare a block here") + } + if len(node.Args) == 0 { + return nil, NodeErr(node, "expected at least one argument") + } + + values := make([]V, 0, len(node.Args)) + for _, arg := range node.Args { + val, ok := mapped[arg] + if !ok { + validValues := make([]string, 0, len(mapped)) + for k := range mapped { + validValues = append(validValues, k) + } + return nil, NodeErr(node, "invalid argument, valid values are: %v", validValues) + } + values = append(values, val) + } + return values, nil + }, store) +} + +// Duration maps configuration directive to a time.Duration variable. +// +// Directive must be in form 'name duration' where duration is any string accepted by +// time.ParseDuration. As an additional requirement, result of time.ParseDuration must not +// be negative. +// +// Note that for convenience, if directive does have multiple arguments, they will be joined +// without separators. E.g. 'name 1h 2m' will become 'name 1h2m' and so '1h2m' will be passed +// to time.ParseDuration. +// +// See Map.Custom for description of arguments. +func (m *Map) Duration(name string, inheritGlobal, required bool, defaultVal time.Duration, store *time.Duration) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + if len(node.Args) == 0 { + return nil, NodeErr(node, "at least one argument is required") + } + + durationStr := strings.Join(node.Args, "") + dur, err := time.ParseDuration(durationStr) + if err != nil { + return nil, NodeErr(node, "%v", err) + } + + if dur < 0 { + return nil, NodeErr(node, "duration must not be negative") + } + + return dur, nil + }, store) +} + +func ParseDataSize(s string) (int, error) { + if len(s) == 0 { + return 0, errors.New("missing a number") + } + + // ' ' terminates the number+suffix pair. + s = s + " " + + var total int + currentDigit := "" + suffix := "" + for _, ch := range s { + if unicode.IsDigit(ch) { + if suffix != "" { + return 0, errors.New("unexpected digit after a suffix") + } + currentDigit += string(ch) + continue + } + if ch != ' ' { + suffix += string(ch) + continue + } + + num, err := strconv.Atoi(currentDigit) + if err != nil { + return 0, err + } + + if num < 0 { + return 0, errors.New("value must not be negative") + } + + switch suffix { + case "G": + total += num * 1024 * 1024 * 1024 + case "M": + total += num * 1024 * 1024 + case "K": + total += num * 1024 + case "B", "b": + total += num + default: + if num != 0 { + return 0, errors.New("unknown unit suffix: " + suffix) + } + } + + suffix = "" + currentDigit = "" + } + + return total, nil +} + +// DataSize maps configuration directive to a int variable, representing data size. +// +// Syntax requires unit suffix to be added to the end of string to specify +// data unit and allows multiple arguments (they will be added together). +// +// See Map.Custom for description of arguments. +func (m *Map) DataSize(name string, inheritGlobal, required bool, defaultVal int64, store *int64) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + if len(node.Args) == 0 { + return nil, NodeErr(node, "at least one argument is required") + } + + durationStr := strings.Join(node.Args, " ") + dur, err := ParseDataSize(durationStr) + if err != nil { + return nil, NodeErr(node, "%v", err) + } + + return int64(dur), nil + }, store) +} + +func ParseBool(s string) (bool, error) { + switch strings.ToLower(s) { + case "1", "true", "on", "yes": + return true, nil + case "0", "false", "off", "no": + return false, nil + } + return false, fmt.Errorf("bool argument should be 'yes' or 'no'") +} + +// Bool maps presence of some configuration directive to a boolean variable. +// Additionally, 'name yes' and 'name no' are mapped to true and false +// correspondingly. +// +// I.e. if directive 'io_debug' exists in processed configuration block or in +// the global configuration (if inheritGlobal is true) then Process will store +// true in target variable. +func (m *Map) Bool(name string, inheritGlobal, defaultVal bool, store *bool) { + m.Custom(name, inheritGlobal, false, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + if len(node.Args) == 0 { + return true, nil + } + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected exactly 1 argument") + } + + b, err := ParseBool(node.Args[0]) + if err != nil { + return nil, NodeErr(node, "bool argument should be 'yes' or 'no'") + } + return b, nil + }, store) +} + +// StringList maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name arbitrary_string arbitrary_string ...' +// Where at least one argument must be present. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) StringList(name string, inheritGlobal, required bool, defaultVal []string, store *[]string) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) == 0 { + return nil, NodeErr(node, "expected at least one argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + return node.Args, nil + }, store) +} + +// String maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name arbitrary_string'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) String(name string, inheritGlobal, required bool, defaultVal string, store *string) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + return node.Args[0], nil + }, store) +} + +// Int maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name 123'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) Int(name string, inheritGlobal, required bool, defaultVal int, store *int) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + i, err := strconv.Atoi(node.Args[0]) + if err != nil { + return nil, NodeErr(node, "invalid integer: %s", node.Args[0]) + } + return i, nil + }, store) +} + +// UInt maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name 123'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) UInt(name string, inheritGlobal, required bool, defaultVal uint, store *uint) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + i, err := strconv.ParseUint(node.Args[0], 10, 32) + if err != nil { + return nil, NodeErr(node, "invalid integer: %s", node.Args[0]) + } + return uint(i), nil + }, store) +} + +// Int32 maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name 123'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) Int32(name string, inheritGlobal, required bool, defaultVal int32, store *int32) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + i, err := strconv.ParseInt(node.Args[0], 10, 32) + if err != nil { + return nil, NodeErr(node, "invalid integer: %s", node.Args[0]) + } + return int32(i), nil + }, store) +} + +// UInt32 maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name 123'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) UInt32(name string, inheritGlobal, required bool, defaultVal uint32, store *uint32) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + i, err := strconv.ParseUint(node.Args[0], 10, 32) + if err != nil { + return nil, NodeErr(node, "invalid integer: %s", node.Args[0]) + } + return uint32(i), nil + }, store) +} + +// Int64 maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name 123'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) Int64(name string, inheritGlobal, required bool, defaultVal int64, store *int64) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + i, err := strconv.ParseInt(node.Args[0], 10, 64) + if err != nil { + return nil, NodeErr(node, "invalid integer: %s", node.Args[0]) + } + return i, nil + }, store) +} + +// UInt64 maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name 123'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) UInt64(name string, inheritGlobal, required bool, defaultVal uint64, store *uint64) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + if len(node.Children) != 0 { + return nil, NodeErr(node, "can't declare block here") + } + + i, err := strconv.ParseUint(node.Args[0], 10, 64) + if err != nil { + return nil, NodeErr(node, "invalid integer: %s", node.Args[0]) + } + return i, nil + }, store) +} + +// Float maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// Configuration directive must be in form 'name 123.55'. +// +// See Custom function for details about inheritGlobal, required and +// defaultVal. +func (m *Map) Float(name string, inheritGlobal, required bool, defaultVal float64, store *float64) { + m.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, func(_ *Map, node Node) (interface{}, error) { + if len(node.Args) != 1 { + return nil, NodeErr(node, "expected 1 argument") + } + + f, err := strconv.ParseFloat(node.Args[0], 64) + if err != nil { + return nil, NodeErr(node, "invalid float: %s", node.Args[0]) + } + return f, nil + }, store) +} + +// Custom maps configuration directive with the specified name to variable +// referenced by 'store' pointer. +// +// If inheritGlobal is true - Map will try to use a value from globalCfg if +// none is set in a processed configuration block. +// +// If required is true - Map will fail if no value is set in the configuration, +// both global (if inheritGlobal is true) and in the processed block. +// +// defaultVal is a factory function that should return the default value for +// the variable. It will be used if no value is set in the config. It can be +// nil if required is true. +// Note that if inheritGlobal is true, defaultVal of the global directive +// will be used instead. +// +// mapper is a function that should convert configuration directive arguments +// into variable value. Both functions may fail with errors, configuration +// processing will stop immediately then. +// Note: mapper function should not modify passed values. +// +// store is where the value returned by mapper should be stored. Can be nil +// (value will be saved only in Map.Values). +func (m *Map) Custom(name string, inheritGlobal, required bool, defaultVal func() (interface{}, error), mapper func(*Map, Node) (interface{}, error), store interface{}) { + if m.entries == nil { + m.entries = make(map[string]matcher) + } + if _, ok := m.entries[name]; ok { + panic("Map.Custom: duplicate matcher") + } + + var target *reflect.Value + ptr := reflect.ValueOf(store) + if ptr.IsValid() && !ptr.IsNil() { + val := ptr.Elem() + if !val.CanSet() { + panic("Map.Custom: store argument must be settable (a pointer)") + } + target = &val + } + + m.entries[name] = matcher{ + name: name, + inheritGlobal: inheritGlobal, + required: required, + defaultVal: defaultVal, + mapper: mapper, + store: target, + } +} + +// Callback creates mapping that will call mapper() function for each +// directive with the specified name. No further processing is done. +// +// Directives with the specified name will not be returned by Process if +// AllowUnknown is used. +// +// It is intended to permit multiple independent values of directive with +// implementation-defined handling. +func (m *Map) Callback(name string, mapper func(*Map, Node) error) { + if m.entries == nil { + m.entries = make(map[string]matcher) + } + if _, ok := m.entries[name]; ok { + panic("Map.Custom: duplicate matcher") + } + + m.entries[name] = matcher{ + name: name, + customCallback: mapper, + } +} + +// Process maps variables from global configuration and block passed in NewMap. +// +// If Map instance was not created using NewMap - Process panics. +func (m *Map) Process() (unknown []Node, err error) { + return m.ProcessWith(m.Globals, m.Block) +} + +// Process maps variables from global configuration and block passed in arguments. +func (m *Map) ProcessWith(globalCfg map[string]interface{}, block Node) (unknown []Node, err error) { + unknown = make([]Node, 0, len(block.Children)) + matched := make(map[string]bool) + m.Values = make(map[string]interface{}) + + for _, subnode := range block.Children { + matcher, ok := m.entries[subnode.Name] + if !ok { + if !m.allowUnknown { + return nil, NodeErr(subnode, "unexpected directive: %s", subnode.Name) + } + unknown = append(unknown, subnode) + continue + } + + if matcher.customCallback != nil { + if err := matcher.customCallback(m, subnode); err != nil { + return nil, err + } + matched[subnode.Name] = true + continue + } + + if matched[subnode.Name] { + return nil, NodeErr(subnode, "duplicate directive: %s", subnode.Name) + } + matched[subnode.Name] = true + + val, err := matcher.mapper(m, subnode) + if err != nil { + return nil, err + } + m.Values[matcher.name] = val + if matcher.store != nil { + matcher.assign(val) + } + } + + for _, matcher := range m.entries { + if matched[matcher.name] { + continue + } + if matcher.mapper == nil { + continue + } + + var val interface{} + globalVal, ok := globalCfg[matcher.name] + if matcher.inheritGlobal && ok { + val = globalVal + } else if !matcher.required { + if matcher.defaultVal == nil { + continue + } + + val, err = matcher.defaultVal() + if err != nil { + return nil, err + } + } else { + return nil, NodeErr(block, "missing required directive: %s", matcher.name) + } + + // If we put zero values into map then code that checks globalCfg + // above will inherit them for required fields instead of failing. + // + // This is important for fields that are required to be specified + // either globally or on per-block basis (e.g. tls, hostname). + // For these directives, global Map does have required = false + // so global values are default which is usually zero value. + // + // This is a temporary solutions, of course, in the long-term + // the way global values and "inheritance" is handled should be + // revised. + store := false + valT := reflect.TypeOf(val) + if valT != nil { + zero := reflect.Zero(valT) + store = !reflect.DeepEqual(val, zero.Interface()) + } + + if store { + m.Values[matcher.name] = val + } + if matcher.store != nil { + matcher.assign(val) + } + } + + return unknown, nil +} diff --git a/framework/config/map_test.go b/framework/config/map_test.go new file mode 100644 index 0000000..fd82630 --- /dev/null +++ b/framework/config/map_test.go @@ -0,0 +1,495 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package config + +import ( + "testing" +) + +func TestMapProcess(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"bar"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo := "" + m.Custom("foo", false, true, nil, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != "bar" { + t.Errorf("Incorrect value stored in variable, want 'bar', got '%s'", foo) + } +} + +func TestMapProcess_MissingRequired(t *testing.T) { + cfg := Node{ + Children: []Node{}, + } + + m := NewMap(nil, cfg) + + foo := "" + m.Custom("foo", false, true, nil, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err == nil { + t.Errorf("Expected failure") + } +} + +func TestMapProcess_InheritGlobal(t *testing.T) { + cfg := Node{ + Children: []Node{}, + } + + m := NewMap(map[string]interface{}{"foo": "bar"}, cfg) + + foo := "" + m.Custom("foo", true, true, nil, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != "bar" { + t.Errorf("Incorrect value stored in variable, want 'bar', got '%s'", foo) + } +} + +func TestMapProcess_InheritGlobal_MissingRequired(t *testing.T) { + cfg := Node{ + Children: []Node{}, + } + + m := NewMap(map[string]interface{}{}, cfg) + + foo := "" + m.Custom("foo", false, true, nil, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err == nil { + t.Errorf("Expected failure") + } +} + +func TestMapProcess_InheritGlobal_Override(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"bar"}, + }, + }, + } + + m := NewMap(map[string]interface{}{}, cfg) + + foo := "" + m.Custom("foo", false, true, nil, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != "bar" { + t.Errorf("Incorrect value stored in variable, want 'bar', got '%s'", foo) + } +} + +func TestMapProcess_DefaultValue(t *testing.T) { + cfg := Node{ + Children: []Node{}, + } + + m := NewMap(nil, cfg) + + foo := "" + m.Custom("foo", false, false, func() (interface{}, error) { + return "bar", nil + }, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != "bar" { + t.Errorf("Incorrect value stored in variable, want 'bar', got '%s'", foo) + } +} + +func TestMapProcess_InheritGlobal_DefaultValue(t *testing.T) { + cfg := Node{ + Children: []Node{}, + } + + m := NewMap(map[string]interface{}{"foo": "baz"}, cfg) + + foo := "" + m.Custom("foo", true, false, func() (interface{}, error) { + return "bar", nil + }, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != "baz" { + t.Errorf("Incorrect value stored in variable, want 'baz', got '%s'", foo) + } + + t.Run("no global", func(t *testing.T) { + _, err := m.ProcessWith(map[string]interface{}{}, cfg) + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != "bar" { + t.Errorf("Incorrect value stored in variable, want 'bar', got '%s'", foo) + } + }) +} + +func TestMapProcess_Duplicate(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"bar"}, + }, + { + Name: "foo", + Args: []string{"bar"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo := "" + m.Custom("foo", false, true, nil, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err == nil { + t.Errorf("Expected failure") + } +} + +func TestMapProcess_Unexpected(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"baz"}, + }, + { + Name: "bar", + Args: []string{"baz"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo := "" + m.Custom("bar", false, true, nil, func(_ *Map, n Node) (interface{}, error) { + return n.Args[0], nil + }, &foo) + + _, err := m.Process() + if err == nil { + t.Errorf("Expected failure") + } + + m.AllowUnknown() + + unknown, err := m.Process() + if err != nil { + t.Errorf("Unexpected failure: %v", err) + } + + if len(unknown) != 1 { + t.Fatalf("Wrong amount of unknown nodes: %v", len(unknown)) + } + + if unknown[0].Name != "foo" { + t.Fatalf("Wrong node in unknown: %v", unknown[0].Name) + } +} + +func TestMapInt(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"1"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo := 0 + m.Int("foo", false, true, 0, &foo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != 1 { + t.Errorf("Incorrect value stored in variable, want 1, got %d", foo) + } +} + +func TestMapInt_Invalid(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"AAAA"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo := 0 + m.Int("foo", false, true, 0, &foo) + + _, err := m.Process() + if err == nil { + t.Errorf("Expected failure") + } +} + +func TestMapFloat(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"1"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo := 0.0 + m.Float("foo", false, true, 0, &foo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if foo != 1.0 { + t.Errorf("Incorrect value stored in variable, want 1, got %v", foo) + } +} + +func TestMapFloat_Invalid(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + Args: []string{"AAAA"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo := 0.0 + m.Float("foo", false, true, 0, &foo) + + _, err := m.Process() + if err == nil { + t.Errorf("Expected failure") + } +} + +func TestMapBool(t *testing.T) { + cfg := Node{ + Children: []Node{ + { + Name: "foo", + }, + { + Name: "bar", + Args: []string{"yes"}, + }, + { + Name: "baz", + Args: []string{"no"}, + }, + }, + } + + m := NewMap(nil, cfg) + + foo, bar, baz, boo := false, false, false, false + m.Bool("foo", false, false, &foo) + m.Bool("bar", false, false, &bar) + m.Bool("baz", false, false, &baz) + m.Bool("boo", false, false, &boo) + + _, err := m.Process() + if err != nil { + t.Fatalf("Unexpected failure: %v", err) + } + + if !foo { + t.Errorf("Incorrect value stored in variable foo, want true, got false") + } + if !bar { + t.Errorf("Incorrect value stored in variable bar, want true, got false") + } + if baz { + t.Errorf("Incorrect value stored in variable baz, want false, got true") + } + if boo { + t.Errorf("Incorrect value stored in variable boo, want false, got true") + } +} + +func TestParseDataSize(t *testing.T) { + check := func(s string, ok bool, expected int) { + val, err := ParseDataSize(s) + if err != nil && ok { + t.Errorf("unexpected parseDataSize('%s') fail: %v", s, err) + return + } + if err == nil && !ok { + t.Errorf("unexpected parseDataSize('%s') success, got %d", s, val) + return + } + if val != expected { + t.Errorf("parseDataSize('%s') != %d", s, expected) + return + } + } + + check("1M", true, 1024*1024) + check("1K", true, 1024) + check("1b", true, 1) + check("1M 5b", true, 1024*1024+5) + check("1M 5K 5b", true, 1024*1024+5*1024+5) + check("0", true, 0) + check("1", false, 0) + check("1d", false, 0) + check("d", false, 0) + check("unrelated", false, 0) + check("1M5b", false, 0) + check("", false, 0) + check("-5M", false, 0) +} + +func TestMap_Callback(t *testing.T) { + called := map[string]int{} + + cfg := Node{ + Children: []Node{ + { + Name: "test2", + Args: []string{"a"}, + }, + { + Name: "test3", + Args: []string{"b"}, + }, + { + Name: "test3", + Args: []string{"b"}, + }, + { + Name: "unrelated", + Args: []string{"b"}, + }, + }, + } + m := NewMap(nil, cfg) + m.Callback("test1", func(*Map, Node) error { + called["test1"]++ + return nil + }) + m.Callback("test2", func(_ *Map, n Node) error { + called["test2"]++ + if n.Args[0] != "a" { + t.Fatal("Wrong n.Args[0] for test2:", n.Args[0]) + } + return nil + }) + m.Callback("test3", func(_ *Map, n Node) error { + called["test3"]++ + if n.Args[0] != "b" { + t.Fatal("Wrong n.Args[0] for test2:", n.Args[0]) + } + return nil + }) + m.AllowUnknown() + others, err := m.Process() + if err != nil { + t.Fatal("Unexpected error:", err) + } + if called["test1"] != 0 { + t.Error("test1 CB was called when it should not") + } + if called["test2"] != 1 { + t.Error("test2 CB was not called when it should") + } + if called["test3"] != 2 { + t.Error("test3 CB was not called when it should") + } + if len(others) != 1 { + t.Error("Wrong amount of unmatched directives") + } + if others[0].Name != "unrelated" { + t.Error("Wrong directive returned in unmatched slice:", others[0].Name) + } +} diff --git a/framework/config/module/check_action.go b/framework/config/module/check_action.go new file mode 100644 index 0000000..cac3278 --- /dev/null +++ b/framework/config/module/check_action.go @@ -0,0 +1,184 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package modconfig + +import ( + "errors" + "fmt" + "strconv" + "strings" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" +) + +// FailAction specifies actions that messages pipeline should take based on the +// result of the check. +// +// Its check module responsibility to apply FailAction on the CheckResult it +// returns. It is intended to be used as follows: +// +// Add the configuration directive to allow user to specify the action: +// +// cfg.Custom("SOME_action", false, false, +// func() (interface{}, error) { +// return modconfig.FailAction{Quarantine: true}, nil +// }, modconfig.FailActionDirective, &yourModule.SOMEAction) +// +// return in func literal is the default value, you might want to adjust it. +// +// Call yourModule.SOMEAction.Apply on CheckResult containing only the +// Reason field: +// +// func (yourModule YourModule) CheckConnection() module.CheckResult { +// return yourModule.SOMEAction.Apply(module.CheckResult{ +// Reason: ..., +// }) +// } +type FailAction struct { + Quarantine bool + Reject bool + + ReasonOverride *exterrors.SMTPError +} + +func FailActionDirective(_ *config.Map, node config.Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, config.NodeErr(node, "can't declare block here") + } + + val, err := ParseActionDirective(node.Args) + if err != nil { + return nil, config.NodeErr(node, "%v", err) + } + return val, nil +} + +func ParseActionDirective(args []string) (FailAction, error) { + if len(args) == 0 { + return FailAction{}, errors.New("expected at least 1 argument") + } + + res := FailAction{} + + switch args[0] { + case "reject", "quarantine": + if len(args) > 1 { + var err error + res.ReasonOverride, err = ParseRejectDirective(args[1:]) + if err != nil { + return FailAction{}, err + } + } + case "ignore": + default: + return FailAction{}, errors.New("invalid action") + } + + res.Reject = args[0] == "reject" + res.Quarantine = args[0] == "quarantine" + return res, nil +} + +// Apply merges the result of check execution with action configuration specified +// in the check configuration. +func (cfa FailAction) Apply(originalRes module.CheckResult) module.CheckResult { + if originalRes.Reason == nil { + return originalRes + } + + if cfa.ReasonOverride != nil { + // Wrap instead of replace to preserve other fields. + originalRes.Reason = &exterrors.SMTPError{ + Code: cfa.ReasonOverride.Code, + EnhancedCode: cfa.ReasonOverride.EnhancedCode, + Message: cfa.ReasonOverride.Message, + Err: originalRes.Reason, + } + } + + originalRes.Quarantine = cfa.Quarantine || originalRes.Quarantine + originalRes.Reject = cfa.Reject || originalRes.Reject + return originalRes +} + +func ParseRejectDirective(args []string) (*exterrors.SMTPError, error) { + code := 554 + enchCode := exterrors.EnhancedCode{0, 7, 0} + msg := "Message rejected due to a local policy" + var err error + switch len(args) { + case 3: + msg = args[2] + if msg == "" { + return nil, fmt.Errorf("message can't be empty") + } + fallthrough + case 2: + enchCode, err = parseEnhancedCode(args[1]) + if err != nil { + return nil, err + } + if enchCode[0] != 4 && enchCode[0] != 5 { + return nil, fmt.Errorf("enhanced code should use either 4 or 5 as a first number") + } + fallthrough + case 1: + code, err = strconv.Atoi(args[0]) + if err != nil { + return nil, fmt.Errorf("invalid error code integer: %v", err) + } + if (code/100) != 4 && (code/100) != 5 { + return nil, fmt.Errorf("error code should start with either 4 or 5") + } + // If enchanced code is not set - set first digit based on provided "basic" code. + if enchCode[0] == 0 { + enchCode[0] = code / 100 + } + case 0: + // If no codes provided at all - use 5.7.0 and 554. + enchCode[0] = 5 + default: + return nil, fmt.Errorf("invalid count of arguments") + } + return &exterrors.SMTPError{ + Code: code, + EnhancedCode: enchCode, + Message: msg, + Reason: "reject directive used", + }, nil +} + +func parseEnhancedCode(s string) (exterrors.EnhancedCode, error) { + parts := strings.Split(s, ".") + if len(parts) != 3 { + return exterrors.EnhancedCode{}, fmt.Errorf("wrong amount of enhanced code parts") + } + + code := exterrors.EnhancedCode{} + for i, part := range parts { + num, err := strconv.Atoi(part) + if err != nil { + return code, err + } + code[i] = num + } + return code, nil +} diff --git a/framework/config/module/interfaces.go b/framework/config/module/interfaces.go new file mode 100644 index 0000000..caf17c5 --- /dev/null +++ b/framework/config/module/interfaces.go @@ -0,0 +1,98 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package modconfig + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +func MessageCheck(globals map[string]interface{}, args []string, block config.Node) (module.Check, error) { + var check module.Check + if err := ModuleFromNode("check", args, block, globals, &check); err != nil { + return nil, err + } + return check, nil +} + +// DeliveryDirective is a callback for use in config.Map.Custom. +// +// It does all work necessary to create a module instance from the config +// directive with the following structure: +// +// directive_name mod_name [inst_name] [{ +// inline_mod_config +// }] +// +// Note that if used configuration structure lacks directive_name before mod_name - this function +// should not be used (call DeliveryTarget directly). +func DeliveryDirective(m *config.Map, node config.Node) (interface{}, error) { + return DeliveryTarget(m.Globals, node.Args, node) +} + +func DeliveryTarget(globals map[string]interface{}, args []string, block config.Node) (module.DeliveryTarget, error) { + var target module.DeliveryTarget + if err := ModuleFromNode("target", args, block, globals, &target); err != nil { + return nil, err + } + return target, nil +} + +func MsgModifier(globals map[string]interface{}, args []string, block config.Node) (module.Modifier, error) { + var check module.Modifier + if err := ModuleFromNode("modify", args, block, globals, &check); err != nil { + return nil, err + } + return check, nil +} + +func IMAPFilter(globals map[string]interface{}, args []string, block config.Node) (module.IMAPFilter, error) { + var filter module.IMAPFilter + if err := ModuleFromNode("imap.filter", args, block, globals, &filter); err != nil { + return nil, err + } + return filter, nil +} + +func StorageDirective(m *config.Map, node config.Node) (interface{}, error) { + var backend module.Storage + if err := ModuleFromNode("storage", node.Args, node, m.Globals, &backend); err != nil { + return nil, err + } + return backend, nil +} + +// Table is a convenience wrapper for TableDirective. +// +// cfg.Bool(...) +// modconfig.Table(cfg, "auth_map", false, false, nil, &mod.authMap) +// cfg.Process() +func Table(cfg *config.Map, name string, inheritGlobal, required bool, defaultVal module.Table, store *module.Table) { + cfg.Custom(name, inheritGlobal, required, func() (interface{}, error) { + return defaultVal, nil + }, TableDirective, store) +} + +func TableDirective(m *config.Map, node config.Node) (interface{}, error) { + var tbl module.Table + if err := ModuleFromNode("table", node.Args, node, m.Globals, &tbl); err != nil { + return nil, err + } + return tbl, nil +} diff --git a/framework/config/module/modconfig.go b/framework/config/module/modconfig.go new file mode 100644 index 0000000..3183bb9 --- /dev/null +++ b/framework/config/module/modconfig.go @@ -0,0 +1,163 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package modconfig provides matchers for config.Map that query +// modules registry and parse inline module definitions. +// +// They should be used instead of manual querying when there is need to +// reference a module instance in the configuration. +// +// See ModuleFromNode documentation for explanation of what is 'args' +// for some functions (DeliveryTarget). +package modconfig + +import ( + "fmt" + "io" + "reflect" + "strings" + + parser "github.com/foxcpp/maddy/framework/cfgparser" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +// createInlineModule is a helper function for config matchers that can create inline modules. +func createInlineModule(preferredNamespace, modName string, args []string) (module.Module, error) { + var newMod module.FuncNewModule + originalModName := modName + + // First try to extend the name with preferred namespace unless the name + // already contains it. + if !strings.Contains(modName, ".") && preferredNamespace != "" { + modName = preferredNamespace + "." + modName + newMod = module.Get(modName) + } + + // Then try global namespace for compatibility and complex modules. + if newMod == nil { + newMod = module.Get(originalModName) + } + + // Bail if both failed. + if newMod == nil { + return nil, fmt.Errorf("unknown module: %s (namespace: %s)", originalModName, preferredNamespace) + } + + return newMod(modName, "", nil, args) +} + +// initInlineModule constructs "faked" config tree and passes it to module +// Init function to make it look like it is defined at top-level. +// +// args must contain at least one argument, otherwise initInlineModule panics. +func initInlineModule(modObj module.Module, globals map[string]interface{}, block config.Node) error { + err := modObj.Init(config.NewMap(globals, block)) + if err != nil { + return err + } + + if closer, ok := modObj.(io.Closer); ok { + hooks.AddHook(hooks.EventShutdown, func() { + log.Debugf("close %s (%s)", modObj.Name(), modObj.InstanceName()) + if err := closer.Close(); err != nil { + log.Printf("module %s (%s) close failed: %v", modObj.Name(), modObj.InstanceName(), err) + } + }) + } + + return nil +} + +// ModuleFromNode does all work to create or get existing module object with a certain type. +// It is not used by top-level module definitions, only for references from other +// modules configuration blocks. +// +// inlineCfg should contain configuration directives for inline declarations. +// args should contain values that are used to create module. +// It should be either module name + instance name or just module name. Further extensions +// may add other string arguments (currently, they can be accessed by module instances +// as inlineArgs argument to constructor). +// +// It checks using reflection whether it is possible to store a module object into modObj +// pointer (e.g. it implements all necessary interfaces) and stores it if everything is fine. +// If module object doesn't implement necessary module interfaces - error is returned. +// If modObj is not a pointer, ModuleFromNode panics. +// +// preferredNamespace is used as an implicit prefix for module name lookups. +// Module with name preferredNamespace + "." + args[0] will be preferred over just args[0]. +// It can be omitted. +func ModuleFromNode(preferredNamespace string, args []string, inlineCfg config.Node, globals map[string]interface{}, moduleIface interface{}) error { + if len(args) == 0 { + return parser.NodeErr(inlineCfg, "at least one argument is required") + } + + referenceExisting := strings.HasPrefix(args[0], "&") + + var modObj module.Module + var err error + if referenceExisting { + if len(args) != 1 || inlineCfg.Children != nil { + return parser.NodeErr(inlineCfg, "exactly one argument is required to use existing config block") + } + modObj, err = module.GetInstance(args[0][1:]) + log.Debugf("%s:%d: reference %s", inlineCfg.File, inlineCfg.Line, args[0]) + } else { + log.Debugf("%s:%d: new module %s %v", inlineCfg.File, inlineCfg.Line, args[0], args[1:]) + modObj, err = createInlineModule(preferredNamespace, args[0], args[1:]) + } + if err != nil { + return err + } + + // NOTE: This will panic if moduleIface is not a pointer. + modIfaceType := reflect.TypeOf(moduleIface).Elem() + modObjType := reflect.TypeOf(modObj) + + if modIfaceType.Kind() == reflect.Interface { + // Case for assignment to module interface type. + if !modObjType.Implements(modIfaceType) && !modObjType.AssignableTo(modIfaceType) { + return parser.NodeErr(inlineCfg, "module %s (%s) doesn't implement %v interface", modObj.Name(), modObj.InstanceName(), modIfaceType) + } + } else if !modObjType.AssignableTo(modIfaceType) { + // Case for assignment to concrete module type. Used in "module groups". + return parser.NodeErr(inlineCfg, "module %s (%s) is not %v", modObj.Name(), modObj.InstanceName(), modIfaceType) + } + + reflect.ValueOf(moduleIface).Elem().Set(reflect.ValueOf(modObj)) + + if !referenceExisting { + if err := initInlineModule(modObj, globals, inlineCfg); err != nil { + return err + } + } + + return nil +} + +// GroupFromNode provides a special kind of ModuleFromNode syntax that allows +// to omit the module name when defining inine configuration. If it is not +// present, name in defaultModule is used. +func GroupFromNode(defaultModule string, args []string, inlineCfg config.Node, globals map[string]interface{}, moduleIface interface{}) error { + if len(args) == 0 { + args = append(args, defaultModule) + } + return ModuleFromNode("", args, inlineCfg, globals, moduleIface) +} diff --git a/framework/config/tls/client.go b/framework/config/tls/client.go new file mode 100644 index 0000000..cf21b3c --- /dev/null +++ b/framework/config/tls/client.go @@ -0,0 +1,88 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tls + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" +) + +func TLSClientBlock(_ *config.Map, node config.Node) (interface{}, error) { + cfg := tls.Config{} + + childM := config.NewMap(nil, node) + var ( + tlsVersions [2]uint16 + rootCAPaths []string + certPath, keyPath string + ) + + childM.StringList("root_ca", false, false, nil, &rootCAPaths) + childM.String("cert", false, false, "", &certPath) + childM.String("key", false, false, "", &keyPath) + childM.Custom("protocols", false, false, func() (interface{}, error) { + return [2]uint16{0, 0}, nil + }, TLSVersionsDirective, &tlsVersions) + childM.Custom("ciphers", false, false, func() (interface{}, error) { + return nil, nil + }, TLSCiphersDirective, &cfg.CipherSuites) + childM.Custom("curves", false, false, func() (interface{}, error) { + return nil, nil + }, TLSCurvesDirective, &cfg.CurvePreferences) + + if _, err := childM.Process(); err != nil { + return nil, err + } + + if len(rootCAPaths) != 0 { + pool := x509.NewCertPool() + for _, path := range rootCAPaths { + blob, err := os.ReadFile(path) + if err != nil { + return nil, err + } + if !pool.AppendCertsFromPEM(blob) { + return nil, fmt.Errorf("no certificates was loaded from %s", path) + } + } + cfg.RootCAs = pool + } + + if certPath != "" && keyPath != "" { + keypair, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return nil, err + } + log.Debugf("using client keypair %s/%s", certPath, keyPath) + cfg.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return &keypair, nil + } + } + + cfg.MinVersion = tlsVersions[0] + cfg.MaxVersion = tlsVersions[1] + log.Debugf("tls: min version: %x, max version: %x", tlsVersions[0], tlsVersions[1]) + + return &cfg, nil +} diff --git a/framework/config/tls/general.go b/framework/config/tls/general.go new file mode 100644 index 0000000..a3fd953 --- /dev/null +++ b/framework/config/tls/general.go @@ -0,0 +1,136 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tls + +import ( + "crypto/tls" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" +) + +var strVersionsMap = map[string]uint16{ + "tls1.0": tls.VersionTLS10, + "tls1.1": tls.VersionTLS11, + "tls1.2": tls.VersionTLS12, + "tls1.3": tls.VersionTLS13, + "": 0, // use crypto/tls defaults if value is not specified +} + +var strCiphersMap = map[string]uint16{ + // TLS 1.0 - 1.2 cipher suites. + "RSA-WITH-RC4128-SHA": tls.TLS_RSA_WITH_RC4_128_SHA, + "RSA-WITH-3DES-EDE-CBC-SHA": tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, + "RSA-WITH-AES128-CBC-SHA": tls.TLS_RSA_WITH_AES_128_CBC_SHA, + "RSA-WITH-AES256-CBC-SHA": tls.TLS_RSA_WITH_AES_256_CBC_SHA, + "RSA-WITH-AES128-CBC-SHA256": tls.TLS_RSA_WITH_AES_128_CBC_SHA256, + "RSA-WITH-AES128-GCM-SHA256": tls.TLS_RSA_WITH_AES_128_GCM_SHA256, + "RSA-WITH-AES256-GCM-SHA384": tls.TLS_RSA_WITH_AES_256_GCM_SHA384, + "ECDHE-ECDSA-WITH-RC4128-SHA": tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, + "ECDHE-ECDSA-WITH-AES128-CBC-SHA": tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + "ECDHE-ECDSA-WITH-AES256-CBC-SHA": tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + "ECDHE-RSA-WITH-RC4128-SHA": tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, + "ECDHE-RSA-WITH-3DES-EDE-CBC-SHA": tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, + "ECDHE-RSA-WITH-AES128-CBC-SHA": tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + "ECDHE-RSA-WITH-AES256-CBC-SHA": tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + "ECDHE-ECDSA-WITH-AES128-CBC-SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, + "ECDHE-RSA-WITH-AES128-CBC-SHA256": tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, + "ECDHE-RSA-WITH-AES128-GCM-SHA256": tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + "ECDHE-ECDSA-WITH-AES128-GCM-SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + "ECDHE-RSA-WITH-AES256-GCM-SHA384": tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + "ECDHE-ECDSA-WITH-AES256-GCM-SHA384": tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + "ECDHE-RSA-WITH-CHACHA20-POLY1305": tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + "ECDHE-ECDSA-WITH-CHACHA20-POLY1305": tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, +} + +var strCurvesMap = map[string]tls.CurveID{ + "p256": tls.CurveP256, + "p384": tls.CurveP384, + "p521": tls.CurveP521, + "X25519": tls.X25519, +} + +// TLSversionsDirective parses directive with arguments that specify +// minimum and maximum supported TLS versions. +// +// It returns [2]uint16 value for use in corresponding fields from tls.Config. +func TLSVersionsDirective(_ *config.Map, node config.Node) (interface{}, error) { + switch len(node.Args) { + case 1: + value, ok := strVersionsMap[node.Args[0]] + if !ok { + return nil, config.NodeErr(node, "invalid TLS version value: %s", node.Args[0]) + } + return [2]uint16{value, value}, nil + case 2: + minValue, ok := strVersionsMap[node.Args[0]] + if !ok { + return nil, config.NodeErr(node, "invalid TLS version value: %s", node.Args[0]) + } + maxValue, ok := strVersionsMap[node.Args[1]] + if !ok { + return nil, config.NodeErr(node, "invalid TLS version value: %s", node.Args[1]) + } + return [2]uint16{minValue, maxValue}, nil + default: + return nil, config.NodeErr(node, "expected 1 or 2 arguments") + } +} + +// TLSCiphersDirective parses directive with arguments that specify +// list of ciphers to offer to clients (or to use for outgoing connections). +// +// It returns list of []uint16 with corresponding cipher IDs. +func TLSCiphersDirective(_ *config.Map, node config.Node) (interface{}, error) { + if len(node.Args) == 0 { + return nil, config.NodeErr(node, "expected at least 1 argument, got 0") + } + + res := make([]uint16, 0, len(node.Args)) + for _, arg := range node.Args { + cipherId, ok := strCiphersMap[arg] + if !ok { + return nil, config.NodeErr(node, "unknown cipher: %s", arg) + } + res = append(res, cipherId) + } + log.Debugln("tls: using non-default cipherset:", node.Args) + return res, nil +} + +// TLSCurvesDirective parses directive with arguments that specify +// elliptic curves to use during TLS key exchange. +// +// It returns []tls.CurveID. +func TLSCurvesDirective(_ *config.Map, node config.Node) (interface{}, error) { + if len(node.Args) == 0 { + return nil, config.NodeErr(node, "expected at least 1 argument, got 0") + } + + res := make([]tls.CurveID, 0, len(node.Args)) + for _, arg := range node.Args { + curveId, ok := strCurvesMap[arg] + if !ok { + return nil, config.NodeErr(node, "unknown curve: %s", arg) + } + res = append(res, curveId) + } + log.Debugln("tls: using non-default curve preferences:", node.Args) + return res, nil +} diff --git a/framework/config/tls/server.go b/framework/config/tls/server.go new file mode 100644 index 0000000..4fe8e8d --- /dev/null +++ b/framework/config/tls/server.go @@ -0,0 +1,124 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tls + +import ( + "crypto/tls" + + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +type TLSConfig struct { + loader module.TLSLoader + baseCfg *tls.Config +} + +func (cfg *TLSConfig) Get() (*tls.Config, error) { + if cfg.loader == nil { + return nil, nil + } + tlsCfg := cfg.baseCfg.Clone() + + err := cfg.loader.ConfigureTLS(tlsCfg) + if err != nil { + return nil, err + } + + return tlsCfg, nil +} + +// TLSDirective reads the TLS configuration and adds the reload handler to +// reread certificates on SIGUSR2. +// +// The returned value is *tls.Config with GetConfigForClient set. +// If the 'tls off' is used, returned value is nil. +func TLSDirective(m *config.Map, node config.Node) (interface{}, error) { + cfg, err := readTLSBlock(m.Globals, node) + if err != nil { + return nil, err + } + + if cfg == nil { + return nil, nil + } + + return &tls.Config{ + GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + return cfg.Get() + }, + }, nil +} + +func readTLSBlock(globals map[string]interface{}, blockNode config.Node) (*TLSConfig, error) { + baseCfg := tls.Config{ + // Workaround for issue https://github.com/foxcpp/maddy/issues/730 + SessionTicketsDisabled: true, + } + + var loader module.TLSLoader + if len(blockNode.Args) > 0 { + if blockNode.Args[0] == "off" { + return nil, nil + } + + err := modconfig.ModuleFromNode("tls.loader", blockNode.Args, config.Node{}, globals, &loader) + if err != nil { + return nil, err + } + } + + childM := config.NewMap(globals, blockNode) + var tlsVersions [2]uint16 + + childM.Custom("loader", false, false, func() (interface{}, error) { + return loader, nil + }, func(_ *config.Map, node config.Node) (interface{}, error) { + var l module.TLSLoader + err := modconfig.ModuleFromNode("tls.loader", node.Args, node, globals, &l) + return l, err + }, &loader) + + childM.Custom("protocols", false, false, func() (interface{}, error) { + return [2]uint16{tls.VersionTLS10, 0}, nil + }, TLSVersionsDirective, &tlsVersions) + + childM.Custom("ciphers", false, false, func() (interface{}, error) { + return nil, nil + }, TLSCiphersDirective, &baseCfg.CipherSuites) + + childM.Custom("curves", false, false, func() (interface{}, error) { + return nil, nil + }, TLSCurvesDirective, &baseCfg.CurvePreferences) + + if _, err := childM.Process(); err != nil { + return nil, err + } + + baseCfg.MinVersion = tlsVersions[0] + baseCfg.MaxVersion = tlsVersions[1] + log.Debugf("tls: min version: %x, max version: %x", tlsVersions[0], tlsVersions[1]) + + return &TLSConfig{ + loader: loader, + baseCfg: &baseCfg, + }, nil +} diff --git a/framework/dns/debugflags.go b/framework/dns/debugflags.go new file mode 100644 index 0000000..fde218b --- /dev/null +++ b/framework/dns/debugflags.go @@ -0,0 +1,36 @@ +//go:build debugflags +// +build debugflags + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dns + +import ( + maddycli "github.com/foxcpp/maddy/internal/cli" + "github.com/urfave/cli/v2" +) + +func init() { + maddycli.AddGlobalFlag(&cli.StringFlag{ + Name: "debug.dnsoverride", + Usage: "replace the DNS resolver address", + Value: "system-default", + Destination: &overrideServ, + }) +} diff --git a/framework/dns/dnssec.go b/framework/dns/dnssec.go new file mode 100644 index 0000000..b8e9c19 --- /dev/null +++ b/framework/dns/dnssec.go @@ -0,0 +1,403 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dns + +import ( + "context" + "net" + "strconv" + "strings" + "time" + + "github.com/foxcpp/maddy/framework/log" + "github.com/miekg/dns" +) + +type TLSA = dns.TLSA + +// ExtResolver is a convenience wrapper for miekg/dns library that provides +// access to certain low-level functionality (notably, AD flag in responses, +// indicating whether DNSSEC verification was performed by the server). +type ExtResolver struct { + cl *dns.Client + Cfg *dns.ClientConfig +} + +// RCodeError is returned by ExtResolver when the RCODE in response is not +// NOERROR. +type RCodeError struct { + Name string + Code int +} + +func (err RCodeError) Temporary() bool { + return err.Code == dns.RcodeServerFailure +} + +func (err RCodeError) Error() string { + switch err.Code { + case dns.RcodeFormatError: + return "dns: rcode FORMERR when looking up " + err.Name + case dns.RcodeServerFailure: + return "dns: rcode SERVFAIL when looking up " + err.Name + case dns.RcodeNameError: + return "dns: rcode NXDOMAIN when looking up " + err.Name + case dns.RcodeNotImplemented: + return "dns: rcode NOTIMP when looking up " + err.Name + case dns.RcodeRefused: + return "dns: rcode REFUSED when looking up " + err.Name + } + return "dns: non-success rcode: " + strconv.Itoa(err.Code) + " when looking up " + err.Name +} + +func IsNotFound(err error) bool { + if dnsErr, ok := err.(*net.DNSError); ok { + return dnsErr.IsNotFound + } + if rcodeErr, ok := err.(RCodeError); ok { + return rcodeErr.Code == dns.RcodeNameError + } + return false +} + +func isLoopback(addr string) bool { + ip := net.ParseIP(addr) + if ip == nil { + return false + } + return ip.IsLoopback() +} + +func (e ExtResolver) exchange(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + var resp *dns.Msg + var lastErr error + for _, srv := range e.Cfg.Servers { + resp, _, lastErr = e.cl.ExchangeContext(ctx, msg, net.JoinHostPort(srv, e.Cfg.Port)) + if lastErr != nil { + continue + } + + if resp.Rcode != dns.RcodeSuccess { + lastErr = RCodeError{msg.Question[0].Name, resp.Rcode} + continue + } + + // Diregard AD flags from non-local resolvers, likely they are + // communicated with using an insecure channel and so flags can be + // tampered with. + if !isLoopback(srv) { + resp.AuthenticatedData = false + } + + break + } + return resp, lastErr +} + +func (e ExtResolver) AuthLookupAddr(ctx context.Context, addr string) (ad bool, names []string, err error) { + revAddr, err := dns.ReverseAddr(addr) + if err != nil { + return false, nil, err + } + + msg := new(dns.Msg) + msg.SetQuestion(revAddr, dns.TypePTR) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + + resp, err := e.exchange(ctx, msg) + if err != nil { + return false, nil, err + } + + ad = resp.AuthenticatedData + names = make([]string, 0, len(resp.Answer)) + for _, rr := range resp.Answer { + ptrRR, ok := rr.(*dns.PTR) + if !ok { + continue + } + + names = append(names, ptrRR.Ptr) + } + return +} + +func (e ExtResolver) AuthLookupHost(ctx context.Context, host string) (ad bool, addrs []string, err error) { + ad, addrParsed, err := e.AuthLookupIPAddr(ctx, host) + if err != nil { + return false, nil, err + } + + addrs = make([]string, 0, len(addrParsed)) + for _, addr := range addrParsed { + addrs = append(addrs, addr.String()) + } + return ad, addrs, nil +} + +func (e ExtResolver) AuthLookupMX(ctx context.Context, name string) (ad bool, mxs []*net.MX, err error) { + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(name), dns.TypeMX) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + + resp, err := e.exchange(ctx, msg) + if err != nil { + return false, nil, err + } + + ad = resp.AuthenticatedData + mxs = make([]*net.MX, 0, len(resp.Answer)) + for _, rr := range resp.Answer { + mxRR, ok := rr.(*dns.MX) + if !ok { + continue + } + + mxs = append(mxs, &net.MX{ + Host: mxRR.Mx, + Pref: mxRR.Preference, + }) + } + return +} + +func (e ExtResolver) AuthLookupTXT(ctx context.Context, name string) (ad bool, recs []string, err error) { + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(name), dns.TypeTXT) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + + resp, err := e.exchange(ctx, msg) + if err != nil { + return false, nil, err + } + + ad = resp.AuthenticatedData + recs = make([]string, 0, len(resp.Answer)) + for _, rr := range resp.Answer { + txtRR, ok := rr.(*dns.TXT) + if !ok { + continue + } + + recs = append(recs, strings.Join(txtRR.Txt, "")) + } + return +} + +// CheckCNAMEAD is a special function for use in DANE lookups. It attempts to determine final +// (canonical) name of the host and also reports whether the whole chain of CNAME's and final zone +// are "secure". +// +// If there are no A or AAAA records for host, rname = "" is returned. +func (e ExtResolver) CheckCNAMEAD(ctx context.Context, host string) (ad bool, rname string, err error) { + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(host), dns.TypeA) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + resp, err := e.exchange(ctx, msg) + if err != nil { + return false, "", err + } + + for _, r := range resp.Answer { + switch r := r.(type) { + case *dns.A: + rname = r.Hdr.Name + ad = resp.AuthenticatedData // Use AD flag from response we used to determine rname + } + } + + if rname == "" { + // IPv6-only host? Try to find out rname using AAAA lookup. + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(host), dns.TypeAAAA) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + resp, err := e.exchange(ctx, msg) + if err == nil { + for _, r := range resp.Answer { + switch r := r.(type) { + case *dns.AAAA: + rname = r.Hdr.Name + ad = resp.AuthenticatedData + } + } + } + } + + return ad, rname, nil +} + +func (e ExtResolver) AuthLookupCNAME(ctx context.Context, host string) (ad bool, cname string, err error) { + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(host), dns.TypeCNAME) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + resp, err := e.exchange(ctx, msg) + if err != nil { + return false, "", err + } + + for _, r := range resp.Answer { + cnameR, ok := r.(*dns.CNAME) + if !ok { + continue + } + return resp.AuthenticatedData, cnameR.Target, nil + } + + return resp.AuthenticatedData, "", nil +} + +func (e ExtResolver) AuthLookupIPAddr(ctx context.Context, host string) (ad bool, addrs []net.IPAddr, err error) { + // First, query IPv6. + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(host), dns.TypeAAAA) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + + resp, err := e.exchange(ctx, msg) + aaaaFailed := false + var ( + v6ad bool + v6addrs []net.IPAddr + ) + if err != nil { + // Disregard the error for AAAA lookups. + aaaaFailed = true + log.DefaultLogger.Error("Network I/O error during AAAA lookup", err, "host", host) + } else { + v6addrs = make([]net.IPAddr, 0, len(resp.Answer)) + v6ad = resp.AuthenticatedData + for _, rr := range resp.Answer { + aaaaRR, ok := rr.(*dns.AAAA) + if !ok { + continue + } + v6addrs = append(v6addrs, net.IPAddr{IP: aaaaRR.AAAA}) + } + } + + // Then repeat query with IPv4. + msg = new(dns.Msg) + msg.SetQuestion(dns.Fqdn(host), dns.TypeA) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + + resp, err = e.exchange(ctx, msg) + var ( + v4ad bool + v4addrs []net.IPAddr + ) + if err != nil { + if aaaaFailed { + return false, nil, err + } + // Disregard A lookup error if AAAA succeeded. + log.DefaultLogger.Error("Network I/O error during A lookup, using AAAA records", err, "host", host) + } else { + v4ad = resp.AuthenticatedData + v4addrs = make([]net.IPAddr, 0, len(resp.Answer)) + for _, rr := range resp.Answer { + aRR, ok := rr.(*dns.A) + if !ok { + continue + } + v4addrs = append(v4addrs, net.IPAddr{IP: aRR.A}) + } + } + + // A little bit of careful handling is required if AD is inconsistent + // for A and AAAA queries. This unfortunatenly happens in practice. For + // purposes of DANE handling (A/AAAA check) we disregard AAAA records + // if they are not authenctiated and return only A records with AD=true. + + addrs = make([]net.IPAddr, 0, len(v4addrs)+len(v6addrs)) + if !v6ad && !v4ad { + addrs = append(addrs, v6addrs...) + addrs = append(addrs, v4addrs...) + } else { + if v6ad { + addrs = append(addrs, v6addrs...) + } + addrs = append(addrs, v4addrs...) + } + return v4ad, addrs, nil +} + +func (e ExtResolver) AuthLookupTLSA(ctx context.Context, service, network, domain string) (ad bool, recs []TLSA, err error) { + name, err := dns.TLSAName(domain, service, network) + if err != nil { + return false, nil, err + } + + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(name), dns.TypeTLSA) + msg.SetEdns0(4096, false) + msg.AuthenticatedData = true + + resp, err := e.exchange(ctx, msg) + if err != nil { + return false, nil, err + } + + ad = resp.AuthenticatedData + recs = make([]dns.TLSA, 0, len(resp.Answer)) + for _, rr := range resp.Answer { + rr, ok := rr.(*dns.TLSA) + if !ok { + continue + } + + recs = append(recs, *rr) + } + return +} + +func NewExtResolver() (*ExtResolver, error) { + cfg, err := dns.ClientConfigFromFile("/etc/resolv.conf") + if err != nil { + return nil, err + } + + if overrideServ != "" && overrideServ != "system-default" { + host, port, err := net.SplitHostPort(overrideServ) + if err != nil { + panic(err) + } + cfg.Servers = []string{host} + cfg.Port = port + } + + if len(cfg.Servers) == 0 { + cfg.Servers = []string{"127.0.0.1"} + } + + cl := new(dns.Client) + cl.Dialer = &net.Dialer{ + Timeout: time.Duration(cfg.Timeout) * time.Second, + } + return &ExtResolver{ + cl: cl, + Cfg: cfg, + }, nil +} diff --git a/framework/dns/dnssec_test.go b/framework/dns/dnssec_test.go new file mode 100644 index 0000000..774897b --- /dev/null +++ b/framework/dns/dnssec_test.go @@ -0,0 +1,196 @@ +package dns + +import ( + "context" + "fmt" + "net" + "reflect" + "strconv" + "testing" + "time" + + "github.com/foxcpp/maddy/framework/log" + "github.com/miekg/dns" +) + +type TestSrvAction int + +const ( + TestSrvTimeout TestSrvAction = iota + TestSrvServfail + TestSrvNoAddr + TestSrvOk +) + +func (a TestSrvAction) String() string { + switch a { + case TestSrvTimeout: + return "SrvTimeout" + case TestSrvServfail: + return "SrvServfail" + case TestSrvNoAddr: + return "SrvNoAddr" + case TestSrvOk: + return "SrvOk" + default: + panic("wtf action") + } +} + +type IPAddrTestServer struct { + udpServ dns.Server + aAction TestSrvAction + aAD bool + aaaaAction TestSrvAction + aaaaAD bool +} + +func (s *IPAddrTestServer) Run() { + pconn, err := net.ListenPacket("udp4", "127.0.0.1:0") + if err != nil { + panic(err) + } + s.udpServ.PacketConn = pconn + s.udpServ.Handler = s + go s.udpServ.ActivateAndServe() //nolint:errcheck +} + +func (s *IPAddrTestServer) Close() { + s.udpServ.PacketConn.Close() +} + +func (s *IPAddrTestServer) Addr() *net.UDPAddr { + return s.udpServ.PacketConn.LocalAddr().(*net.UDPAddr) +} + +func (s *IPAddrTestServer) ServeDNS(w dns.ResponseWriter, m *dns.Msg) { + q := m.Question[0] + + var ( + act TestSrvAction + ad bool + ) + switch q.Qtype { + case dns.TypeA: + act = s.aAction + ad = s.aAD + case dns.TypeAAAA: + act = s.aaaaAction + ad = s.aaaaAD + default: + panic("wtf qtype") + } + + reply := new(dns.Msg) + reply.SetReply(m) + reply.RecursionAvailable = true + reply.AuthenticatedData = ad + + switch act { + case TestSrvTimeout: + return // no nobody heard from him since... + case TestSrvServfail: + reply.Rcode = dns.RcodeServerFailure + case TestSrvNoAddr: + case TestSrvOk: + switch q.Qtype { + case dns.TypeA: + reply.Answer = append(reply.Answer, &dns.A{ + Hdr: dns.RR_Header{ + Name: q.Name, + Rrtype: dns.TypeA, + Class: dns.ClassINET, + Ttl: 9999, + }, + A: net.ParseIP("127.0.0.1"), + }) + case dns.TypeAAAA: + reply.Answer = append(reply.Answer, &dns.AAAA{ + Hdr: dns.RR_Header{ + Name: q.Name, + Rrtype: dns.TypeAAAA, + Class: dns.ClassINET, + Ttl: 9999, + }, + AAAA: net.ParseIP("::1"), + }) + } + } + + if err := w.WriteMsg(reply); err != nil { + panic(err) + } +} + +func TestExtResolver_AuthLookupIPAddr(t *testing.T) { + // AuthLookupIPAddr has a rather convoluted logic for combined A/AAAA + // lookups that return the best-effort result and also has some nuanced in + // AD flag handling for use in DANE algorithms. + + // Silence log messages about disregarded I/O errors. + log.DefaultLogger.Out = nil + + test := func(aAct, aaaaAct TestSrvAction, aAD, aaaaAD, ad bool, addrs []net.IP, err bool) { + t.Helper() + t.Run(fmt.Sprintln(aAct, aaaaAct, aAD, aaaaAD), func(t *testing.T) { + t.Helper() + + s := IPAddrTestServer{} + s.aAction = aAct + s.aaaaAction = aaaaAct + s.aAD = aAD + s.aaaaAD = aaaaAD + s.Run() + defer s.Close() + res := ExtResolver{ + cl: new(dns.Client), + Cfg: &dns.ClientConfig{ + Servers: []string{"127.0.0.1"}, + Port: strconv.Itoa(s.Addr().Port), + Timeout: 1, + }, + } + res.cl.Dialer = &net.Dialer{ + Timeout: 500 * time.Millisecond, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + actualAd, actualAddrs, actualErr := res.AuthLookupIPAddr(ctx, "maddy.test") + if (actualErr != nil) != err { + t.Fatal("actualErr:", actualErr, "expectedErr:", err) + } + if actualAd != ad { + t.Error("actualAd:", actualAd, "expectedAd:", ad) + } + ipAddrs := make([]net.IPAddr, 0, len(addrs)) + if len(addrs) == 0 { + ipAddrs = nil // lookup returns nil addrs for error cases + } + for _, a := range addrs { + ipAddrs = append(ipAddrs, net.IPAddr{IP: a, Zone: ""}) + } + if !reflect.DeepEqual(actualAddrs, ipAddrs) { + t.Logf("actualAddrs: %#+v", actualAddrs) + t.Logf("addrs: %#+v", ipAddrs) + t.Fail() + } + }) + } + + test(TestSrvOk, TestSrvOk, true, true, true, []net.IP{net.ParseIP("::1"), net.ParseIP("127.0.0.1").To4()}, false) + test(TestSrvOk, TestSrvOk, true, false, true, []net.IP{net.ParseIP("127.0.0.1").To4()}, false) + test(TestSrvOk, TestSrvOk, false, true, false, []net.IP{net.ParseIP("::1"), net.ParseIP("127.0.0.1").To4()}, false) + test(TestSrvOk, TestSrvOk, false, false, false, []net.IP{net.ParseIP("::1"), net.ParseIP("127.0.0.1").To4()}, false) + test(TestSrvOk, TestSrvTimeout, true, true, true, []net.IP{net.ParseIP("127.0.0.1").To4()}, false) + test(TestSrvOk, TestSrvServfail, true, true, true, []net.IP{net.ParseIP("127.0.0.1").To4()}, false) + test(TestSrvOk, TestSrvNoAddr, true, true, true, []net.IP{net.ParseIP("127.0.0.1").To4()}, false) + test(TestSrvNoAddr, TestSrvOk, true, true, true, []net.IP{net.ParseIP("::1")}, false) + test(TestSrvServfail, TestSrvServfail, true, true, false, nil, true) + + // actualAd is false, we don't want to risk reporting positive AD result if + // something is wrong with IPv4 lookup. + test(TestSrvTimeout, TestSrvOk, true, true, false, []net.IP{net.ParseIP("::1")}, false) + test(TestSrvServfail, TestSrvOk, true, true, false, []net.IP{net.ParseIP("::1")}, false) +} diff --git a/framework/dns/idna.go b/framework/dns/idna.go new file mode 100644 index 0000000..3592c42 --- /dev/null +++ b/framework/dns/idna.go @@ -0,0 +1,37 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dns + +import ( + "golang.org/x/net/idna" + "golang.org/x/text/unicode/norm" +) + +// SelectIDNA is a convenience function for encoding to/from Punycode. +// +// If ulabel is true, it returns U-label encoded domain in the Unicode NFC +// form. +// If ulabel is false, it returns A-label encoded domain. +func SelectIDNA(ulabel bool, domain string) (string, error) { + if ulabel { + uDomain, err := idna.ToUnicode(domain) + return norm.NFC.String(uDomain), err + } + return idna.ToASCII(domain) +} diff --git a/framework/dns/norm.go b/framework/dns/norm.go new file mode 100644 index 0000000..6d236e9 --- /dev/null +++ b/framework/dns/norm.go @@ -0,0 +1,72 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dns + +import ( + "strings" + + "github.com/miekg/dns" + "golang.org/x/net/idna" + "golang.org/x/text/unicode/norm" +) + +func FQDN(domain string) string { + return dns.Fqdn(domain) +} + +// ForLookup converts the domain into a canonical form suitable for table +// lookups and other comparisons. +// +// TL;DR Use this instead of strings.ToLower to prepare domain for lookups. +// +// Domains that contain invalid UTF-8 or invalid A-label +// domains are simply converted to local-case using strings.ToLower, but the +// error is also returned. +func ForLookup(domain string) (string, error) { + uDomain, err := idna.ToUnicode(domain) + if err != nil { + return strings.ToLower(domain), err + } + + // Side note: strings.ToLower does not support full case-folding, so it is + // important to apply NFC normalization first. + uDomain = norm.NFC.String(uDomain) + uDomain = strings.ToLower(uDomain) + uDomain = strings.TrimSuffix(uDomain, ".") + return uDomain, nil +} + +// Equal reports whether domain1 and domain2 are equivalent as defined by +// IDNA2008 (RFC 5890). +// +// TL;DR Use this instead of strings.EqualFold to compare domains. +// +// Equivalence for malformed A-label domains is defined using regular +// byte-string comparison with case-folding applied. +func Equal(domain1, domain2 string) bool { + // Short circult. If they are bit-equivalent, then they are also semantically + // equivalent. + if domain1 == domain2 { + return true + } + + uDomain1, _ := ForLookup(domain1) + uDomain2, _ := ForLookup(domain2) + return uDomain1 == uDomain2 +} diff --git a/framework/dns/override.go b/framework/dns/override.go new file mode 100644 index 0000000..25f0da0 --- /dev/null +++ b/framework/dns/override.go @@ -0,0 +1,53 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dns + +import ( + "context" + "net" + "time" +) + +var overrideServ string + +// override globally overrides the used DNS server address with one provided. +// This function is meant only for testing. It should be called before any modules are +// initialized to have full effect. +// +// The server argument is in form of "IP:PORT". It is expected that the server +// will be available both using TCP and UDP on the same port. +func override(server string) { + net.DefaultResolver.PreferGo = true + net.DefaultResolver.Dial = func(ctx context.Context, network, _ string) (net.Conn, error) { + dialer := net.Dialer{ + // This is localhost, it is either running or not. Fail quickly if + // we can't connect. + Timeout: 1 * time.Second, + } + + switch network { + case "udp", "udp4", "udp6": + return dialer.DialContext(ctx, "udp4", server) + case "tcp", "tcp4", "tcp6": + return dialer.DialContext(ctx, "tcp4", server) + default: + panic("OverrideDNS.Dial: unknown network") + } + } +} diff --git a/framework/dns/resolver.go b/framework/dns/resolver.go new file mode 100644 index 0000000..f1393fe --- /dev/null +++ b/framework/dns/resolver.go @@ -0,0 +1,61 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package dns defines interfaces used by maddy modules to perform DNS +// lookups. +// +// Currently, there is only Resolver interface which is implemented +// by dns.DefaultResolver(). In the future, DNSSEC-enabled stub resolver +// implementation will be added here. +package dns + +import ( + "context" + "net" + "strings" +) + +// Resolver is an interface that describes DNS-related methods used by maddy. +// +// It is implemented by dns.DefaultResolver(). Methods behave the same way. +type Resolver interface { + LookupAddr(ctx context.Context, addr string) (names []string, err error) + LookupHost(ctx context.Context, host string) (addrs []string, err error) + LookupMX(ctx context.Context, name string) ([]*net.MX, error) + LookupTXT(ctx context.Context, name string) ([]string, error) + LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) +} + +// LookupAddr is a convenience wrapper for Resolver.LookupAddr. +// +// It returns the first name with trailing dot stripped. +func LookupAddr(ctx context.Context, r Resolver, ip net.IP) (string, error) { + names, err := r.LookupAddr(ctx, ip.String()) + if err != nil || len(names) == 0 { + return "", err + } + return strings.TrimRight(names[0], "."), nil +} + +func DefaultResolver() Resolver { + if overrideServ != "" && overrideServ != "system-default" { + override(overrideServ) + } + + return net.DefaultResolver +} diff --git a/framework/exterrors/dns.go b/framework/exterrors/dns.go new file mode 100644 index 0000000..fd0eeeb --- /dev/null +++ b/framework/exterrors/dns.go @@ -0,0 +1,35 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package exterrors + +import ( + "net" +) + +func UnwrapDNSErr(err error) (reason string, misc map[string]interface{}) { + dnsErr, ok := err.(*net.DNSError) + if !ok { + // Return non-nil in case the user will try to 'extend' it with its own + // values. + return "", map[string]interface{}{} + } + + // Nor server name, nor DNS name are usually useful, so exclude them. + return dnsErr.Err, map[string]interface{}{} +} diff --git a/framework/exterrors/exterrors.go b/framework/exterrors/exterrors.go new file mode 100644 index 0000000..2ed34c5 --- /dev/null +++ b/framework/exterrors/exterrors.go @@ -0,0 +1,22 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package errors defines error-handling and primitives +// used across maddy, notably to pass additional error +// information across module boundaries. +package exterrors diff --git a/framework/exterrors/fields.go b/framework/exterrors/fields.go new file mode 100644 index 0000000..8f3e8ab --- /dev/null +++ b/framework/exterrors/fields.go @@ -0,0 +1,74 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package exterrors + +type fieldsErr interface { + Fields() map[string]interface{} +} + +type unwrapper interface { + Unwrap() error +} + +type fieldsWrap struct { + err error + fields map[string]interface{} +} + +func (fw fieldsWrap) Error() string { + return fw.err.Error() +} + +func (fw fieldsWrap) Unwrap() error { + return fw.err +} + +func (fw fieldsWrap) Fields() map[string]interface{} { + return fw.fields +} + +func Fields(err error) map[string]interface{} { + fields := make(map[string]interface{}, 5) + + for err != nil { + errFields, ok := err.(fieldsErr) + if ok { + for k, v := range errFields.Fields() { + // Outer errors override fields of the inner ones. + // Not the reverse. + if fields[k] != nil { + continue + } + fields[k] = v + } + } + + unwrap, ok := err.(unwrapper) + if !ok { + break + } + err = unwrap.Unwrap() + } + + return fields +} + +func WithFields(err error, fields map[string]interface{}) error { + return fieldsWrap{err: err, fields: fields} +} diff --git a/framework/exterrors/smtp.go b/framework/exterrors/smtp.go new file mode 100644 index 0000000..01e16f4 --- /dev/null +++ b/framework/exterrors/smtp.go @@ -0,0 +1,146 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package exterrors + +import ( + "fmt" + + "github.com/emersion/go-smtp" +) + +type EnhancedCode smtp.EnhancedCode + +func (ec EnhancedCode) FormatLog() string { + return fmt.Sprintf("%d.%d.%d", ec[0], ec[1], ec[2]) +} + +// SMTPError type is a copy of emersion/go-smtp.SMTPError type +// that extends it with Fields method for logging and reporting +// in maddy. It should be used instead of the go-smtp library type for all +// errors. +type SMTPError struct { + // SMTP status code. Most of these codes are overly generic and are barely + // useful. Nonetheless, take a look at the 'Associated basic status code' + // in the SMTP Enhanced Status Codes registry (below), then check RFC 5321 + // (Section 4.3.2) and pick what you like. Stick to 451 and 554 if there are + // no useful codes. + Code int + + // Enhanced SMTP status code. If you are unsure, take a look at + // https://www.iana.org/assignments/smtp-enhanced-status-codes/smtp-enhanced-status-codes.xhtml + EnhancedCode EnhancedCode + + // Error message that should be returned to the SMTP client. + // Usually, it should be a short and generic description of the error + // that excludes any details. Especially, for checks, avoid + // mentioning the exact policy mechanism used to avoid disclosing the + // server configuration details. Don't say "DNS error during DMARC check", + // say "DNS error during policy check". Same goes for network and file I/O + // errors. ESPECIALLY, don't include any configuration variables or object + // identifiers in it. + Message string + + // If the error was generated by a message check + // this field includes module name. + CheckName string + + // If the error was generated by a delivery target + // this field includes module name. + TargetName string + + // If the error was generated by a message modifier + // this field includes module name. + ModifierName string + + // If the error was generated as a result of another + // error - this field contains the original error object. + // + // Err.Error() will be copied into the 'reason' field returned + // by the Fields method unless a different values is specified + // using the Reason field below. + Err error + + // Textual explanation of the actual error reason. Defaults to the + // Err.Error() value if Err is not nil, empty string otherwise. + Reason string + + Misc map[string]interface{} +} + +func (se *SMTPError) Unwrap() error { + return se.Err +} + +func (se *SMTPError) Fields() map[string]interface{} { + ctx := make(map[string]interface{}, len(se.Misc)+3) + for k, v := range se.Misc { + ctx[k] = v + } + ctx["smtp_code"] = se.Code + ctx["smtp_enchcode"] = se.EnhancedCode + ctx["smtp_msg"] = se.Message + if se.CheckName != "" { + ctx["check"] = se.CheckName + } + if se.TargetName != "" { + ctx["target"] = se.TargetName + } + if se.Reason != "" { + ctx["reason"] = se.Reason + } else if se.Err != nil { + ctx["reason"] = se.Err.Error() + } + return ctx +} + +// Temporary reports whether +func (se *SMTPError) Temporary() bool { + return se.Code/100 == 4 +} + +func (se *SMTPError) Error() string { + if se.Reason != "" { + return se.Reason + } + if se.Err != nil { + return se.Err.Error() + } + return se.Message +} + +// SMTPCode is a convenience function that returns one of its arguments +// depending on the result of exterrors.IsTemporary for the specified error +// object. +func SMTPCode(err error, temporaryCode, permanentCode int) int { + if IsTemporary(err) { + return temporaryCode + } + return permanentCode +} + +// SMTPEnchCode is a convenience function changes the first number of the SMTP enhanced +// status code based on the value exterrors.IsTemporary returns for the specified +// error object. +func SMTPEnchCode(err error, code EnhancedCode) EnhancedCode { + if IsTemporary(err) { + code[0] = 4 + } + code[0] = 5 + return code +} diff --git a/framework/exterrors/temporary.go b/framework/exterrors/temporary.go new file mode 100644 index 0000000..1b6db16 --- /dev/null +++ b/framework/exterrors/temporary.go @@ -0,0 +1,74 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package exterrors + +import ( + "errors" +) + +type TemporaryErr interface { + Temporary() bool +} + +// IsTemporaryOrUnspec is similar to IsTemporary except that it returns true +// if error does not have a Temporary() method. Basically, it assumes that +// errors are temporary by default compared to IsTemporary that assumes +// errors are permanent by default. +func IsTemporaryOrUnspec(err error) bool { + var temp TemporaryErr + if errors.As(err, &temp) { + return temp.Temporary() + } + return true +} + +// IsTemporary returns true whether the passed error object +// have a Temporary() method and it returns true. +func IsTemporary(err error) bool { + var temp TemporaryErr + if errors.As(err, &temp) { + return temp.Temporary() + } + return false +} + +type temporaryErr struct { + err error + temp bool +} + +func (t temporaryErr) Unwrap() error { + return t.err +} + +func (t temporaryErr) Error() string { + return t.err.Error() +} + +func (t temporaryErr) Temporary() bool { + return t.temp +} + +// WithTemporary wraps the passed error object with the implementation of the +// Temporary() method that will return the specified value. +// +// Original error value can be obtained using errors.Unwrap. +func WithTemporary(err error, temporary bool) error { + return temporaryErr{err, temporary} +} diff --git a/framework/future/future.go b/framework/future/future.go new file mode 100644 index 0000000..8e44c92 --- /dev/null +++ b/framework/future/future.go @@ -0,0 +1,105 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package future + +import ( + "context" + "runtime/debug" + "sync" + + "github.com/foxcpp/maddy/framework/log" +) + +// The Future object implements a container for (value, error) pair that "will +// be populated later" and allows multiple users to wait for it to be set. +// +// It should not be copied after first use. +type Future struct { + mu sync.RWMutex + set bool + val interface{} + err error + + notify chan struct{} +} + +func New() *Future { + return &Future{notify: make(chan struct{})} +} + +// Set sets the Future (value, error) pair. All currently blocked and future +// Get calls will return it. +func (f *Future) Set(val interface{}, err error) { + if f == nil { + panic("nil future used") + } + + f.mu.Lock() + defer f.mu.Unlock() + + if f.set { + stack := debug.Stack() + log.Println("Future.Set called multiple times", stack) + log.Println("value=", val, "err=", err) + return + } + + f.set = true + f.val = val + f.err = err + + close(f.notify) +} + +func (f *Future) Get() (interface{}, error) { + if f == nil { + panic("nil future used") + } + + return f.GetContext(context.Background()) +} + +func (f *Future) GetContext(ctx context.Context) (interface{}, error) { + if f == nil { + panic("nil future used") + } + + f.mu.RLock() + if f.set { + val := f.val + err := f.err + f.mu.RUnlock() + return val, err + } + + f.mu.RUnlock() + select { + case <-f.notify: + case <-ctx.Done(): + return nil, ctx.Err() + } + + f.mu.RLock() + defer f.mu.RUnlock() + if !f.set { + panic("future: Notification received, but value is not set") + } + + return f.val, f.err +} diff --git a/framework/future/future_test.go b/framework/future/future_test.go new file mode 100644 index 0000000..aa0d581 --- /dev/null +++ b/framework/future/future_test.go @@ -0,0 +1,75 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package future + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestFuture_SetBeforeGet(t *testing.T) { + f := New() + + f.Set(1, errors.New("1")) + val, err := f.Get() + if err.Error() != "1" { + t.Error("Wrong error:", err) + } + + if val, _ := val.(int); val != 1 { + t.Fatal("wrong val received from Get") + } +} + +func TestFuture_Wait(t *testing.T) { + f := New() + + go func() { + time.Sleep(500 * time.Millisecond) + f.Set(1, errors.New("1")) + }() + + val, err := f.Get() + if val, _ := val.(int); val != 1 { + t.Fatal("wrong val received from Get") + } + if err.Error() != "1" { + t.Error("Wrong error:", err) + } + + val, err = f.Get() + if val, _ := val.(int); val != 1 { + t.Fatal("wrong val received from Get on second try") + } + if err.Error() != "1" { + t.Error("Wrong error:", err) + } +} + +func TestFuture_WaitCtx(t *testing.T) { + f := New() + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + _, err := f.GetContext(ctx) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatal("context is not cancelled") + } +} diff --git a/framework/hooks/hooks.go b/framework/hooks/hooks.go new file mode 100644 index 0000000..7319fff --- /dev/null +++ b/framework/hooks/hooks.go @@ -0,0 +1,80 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package hooks + +import "sync" + +type Event int + +const ( + // EventShutdown is triggered when the server process is about to stop. + EventShutdown Event = iota + + // EventReload is triggered when the server process receives the SIGUSR2 + // signal (on POSIX platforms) and indicates the request to reload the + // server configuration from persistent storage. + // + // Since it is by design problematic to reload the modules configuration, + // this event only applies to secondary files such as aliases mapping and + // TLS certificates. + EventReload + + // EventLogRotate is triggered when the server process receives the SIGUSR1 + // signal (on POSIX platforms) and indicates the request to reopen used log + // files since they might have rotated. + EventLogRotate +) + +var ( + hooks = make(map[Event][]func()) + hooksLck sync.Mutex +) + +func hooksToRun(eventName Event) []func() { + hooksLck.Lock() + defer hooksLck.Unlock() + hooksEv := hooks[eventName] + if hooksEv == nil { + return nil + } + + // The slice is copied so hooks can be run without holding the lock what + // might be important since they are likely to do a lot of I/O. + hooksEvCpy := make([]func(), 0, len(hooksEv)) + hooksEvCpy = append(hooksEvCpy, hooksEv...) + + return hooksEvCpy +} + +// RunHooks runs the hooks installed for the specified eventName in the reverse +// order. +func RunHooks(eventName Event) { + hooks := hooksToRun(eventName) + for i := len(hooks) - 1; i >= 0; i-- { + hooks[i]() + } +} + +// AddHook installs the hook to be executed when certain event occurs. +func AddHook(eventName Event, f func()) { + hooksLck.Lock() + defer hooksLck.Unlock() + + hooks[eventName] = append(hooks[eventName], f) +} diff --git a/framework/log/log.go b/framework/log/log.go new file mode 100644 index 0000000..f9092a4 --- /dev/null +++ b/framework/log/log.go @@ -0,0 +1,237 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package log implements a minimalistic logging library. +package log + +import ( + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/foxcpp/maddy/framework/exterrors" + "go.uber.org/zap" +) + +// Logger is the structure that writes formatted output to the underlying +// log.Output object. +// +// Logger is stateless and can be copied freely. However, consider that +// underlying log.Output will not be copied. +// +// Each log message is prefixed with logger name. Timestamp and debug flag +// formatting is done by log.Output. +// +// No serialization is provided by Logger, its log.Output responsibility to +// ensure goroutine-safety if necessary. +type Logger struct { + Out Output + Name string + Debug bool + + // Additional fields that will be added + // to the Msg output. + Fields map[string]interface{} +} + +func (l Logger) Zap() *zap.Logger { + // TODO: Migrate to using zap natively. + return zap.New(zapLogger{L: l}) +} + +func (l Logger) Debugf(format string, val ...interface{}) { + if !l.Debug { + return + } + l.log(true, l.formatMsg(fmt.Sprintf(format, val...), nil)) +} + +func (l Logger) Debugln(val ...interface{}) { + if !l.Debug { + return + } + l.log(true, l.formatMsg(strings.TrimRight(fmt.Sprintln(val...), "\n"), nil)) +} + +func (l Logger) Printf(format string, val ...interface{}) { + l.log(false, l.formatMsg(fmt.Sprintf(format, val...), nil)) +} + +func (l Logger) Println(val ...interface{}) { + l.log(false, l.formatMsg(strings.TrimRight(fmt.Sprintln(val...), "\n"), nil)) +} + +// Msg writes an event log message in a machine-readable format (currently +// JSON). +// +// name: msg\t{"key":"value","key2":"value2"} +// +// Key-value pairs are built from fields slice which should contain key strings +// followed by corresponding values. That is, for example, []interface{"key", +// "value", "key2", "value2"}. +// +// If value in fields implements LogFormatter, it will be represented by the +// string returned by FormatLog method. Same goes for fmt.Stringer and error +// interfaces. +// +// Additionally, time.Time is written as a string in ISO 8601 format. +// time.Duration follows fmt.Stringer rule above. +func (l Logger) Msg(msg string, fields ...interface{}) { + m := make(map[string]interface{}, len(fields)/2) + fieldsToMap(fields, m) + l.log(false, l.formatMsg(msg, m)) +} + +// Error writes an event log message in a machine-readable format (currently +// JSON) containing information about the error. If err does have a Fields +// method that returns map[string]interface{}, its result will be added to the +// message. +// +// name: msg\t{"key":"value","key2":"value2"} +// +// Additionally, values from fields will be added to it, as handled by +// Logger.Msg. +// +// In the context of Error method, "msg" typically indicates the top-level +// context in which the error is *handled*. For example, if error leads to +// rejection of SMTP DATA command, msg will probably be "DATA error". +func (l Logger) Error(msg string, err error, fields ...interface{}) { + if err == nil { + return + } + + errFields := exterrors.Fields(err) + allFields := make(map[string]interface{}, len(fields)+len(errFields)+2) + for k, v := range errFields { + allFields[k] = v + } + + // If there is already a 'reason' field - use it, it probably + // provides a better explanation than error text itself. + if allFields["reason"] == nil { + allFields["reason"] = err.Error() + } + fieldsToMap(fields, allFields) + + l.log(false, l.formatMsg(msg, allFields)) +} + +func (l Logger) DebugMsg(kind string, fields ...interface{}) { + if !l.Debug { + return + } + m := make(map[string]interface{}, len(fields)/2) + fieldsToMap(fields, m) + l.log(true, l.formatMsg(kind, m)) +} + +func fieldsToMap(fields []interface{}, out map[string]interface{}) { + var lastKey string + for i, val := range fields { + if i%2 == 0 { + // Key + key, ok := val.(string) + if !ok { + // Misformatted arguments, attempt to provide useful message + // anyway. + out[fmt.Sprint("field", i)] = key + continue + } + lastKey = key + } else { + // Value + out[lastKey] = val + } + } +} + +func (l Logger) formatMsg(msg string, fields map[string]interface{}) string { + formatted := strings.Builder{} + + formatted.WriteString(msg) + formatted.WriteRune('\t') + + if len(l.Fields)+len(fields) != 0 { + if fields == nil { + fields = make(map[string]interface{}) + } + for k, v := range l.Fields { + fields[k] = v + } + if err := marshalOrderedJSON(&formatted, fields); err != nil { + // Fallback to printing the message with minimal processing. + return fmt.Sprintf("[BROKEN FORMATTING: %v] %v %+v", err, msg, fields) + } + } + + return formatted.String() +} + +type LogFormatter interface { + FormatLog() string +} + +// Write implements io.Writer, all bytes sent +// to it will be written as a separate log messages. +// No line-buffering is done. +func (l Logger) Write(s []byte) (int, error) { + l.log(false, strings.TrimRight(string(s), "\n")) + return len(s), nil +} + +// DebugWriter returns a writer that will act like Logger.Write +// but will use debug flag on messages. If Logger.Debug is false, +// Write method of returned object will be no-op. +func (l Logger) DebugWriter() io.Writer { + if !l.Debug { + return io.Discard + } + l.Debug = true + return &l +} + +func (l Logger) log(debug bool, s string) { + if l.Name != "" { + s = l.Name + ": " + s + } + + if l.Out != nil { + l.Out.Write(time.Now(), debug, s) + return + } + if DefaultLogger.Out != nil { + DefaultLogger.Out.Write(time.Now(), debug, s) + return + } + + // Logging is disabled - do nothing. +} + +// DefaultLogger is the global Logger object that is used by +// package-level logging functions. +// +// As with all other Loggers, it is not gorountine-safe on its own, +// however underlying log.Output may provide necessary serialization. +var DefaultLogger = Logger{Out: WriterOutput(os.Stderr, false)} + +func Debugf(format string, val ...interface{}) { DefaultLogger.Debugf(format, val...) } +func Debugln(val ...interface{}) { DefaultLogger.Debugln(val...) } +func Printf(format string, val ...interface{}) { DefaultLogger.Printf(format, val...) } +func Println(val ...interface{}) { DefaultLogger.Println(val...) } diff --git a/framework/log/orderedjson.go b/framework/log/orderedjson.go new file mode 100644 index 0000000..bbe01c2 --- /dev/null +++ b/framework/log/orderedjson.go @@ -0,0 +1,85 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package log + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +// To support ad-hoc parsing in a better way we want to make order of fields in +// output JSON documents determistics. Additionally, this will make them more +// human-readable when values from multiple messages are lined up to each +// other. + +type module interface { + Name() string + InstanceName() string +} + +func marshalOrderedJSON(output *strings.Builder, m map[string]interface{}) error { + order := make([]string, 0, len(m)) + for k := range m { + order = append(order, k) + } + sort.Strings(order) + + output.WriteRune('{') + for i, key := range order { + if i != 0 { + output.WriteRune(',') + } + + jsonKey, err := json.Marshal(key) + if err != nil { + return err + } + + output.Write(jsonKey) + output.WriteString(":") + + val := m[key] + switch casted := val.(type) { + case time.Time: + val = casted.Format("2006-01-02T15:04:05.000") + case time.Duration: + val = casted.String() + case LogFormatter: + val = casted.FormatLog() + case fmt.Stringer: + val = casted.String() + case module: + val = casted.Name() + "/" + casted.InstanceName() + case error: + val = casted.Error() + } + + jsonValue, err := json.Marshal(val) + if err != nil { + return err + } + output.Write(jsonValue) + } + output.WriteRune('}') + + return nil +} diff --git a/framework/log/output.go b/framework/log/output.go new file mode 100644 index 0000000..612a4bf --- /dev/null +++ b/framework/log/output.go @@ -0,0 +1,74 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package log + +import ( + "time" +) + +type Output interface { + Write(stamp time.Time, debug bool, msg string) + Close() error +} + +type multiOut struct { + outs []Output +} + +func (m multiOut) Write(stamp time.Time, debug bool, msg string) { + for _, out := range m.outs { + out.Write(stamp, debug, msg) + } +} + +func (m multiOut) Close() error { + for _, out := range m.outs { + if err := out.Close(); err != nil { + return err + } + } + return nil +} + +func MultiOutput(outputs ...Output) Output { + return multiOut{outputs} +} + +type funcOut struct { + out func(time.Time, bool, string) + close func() error +} + +func (f funcOut) Write(stamp time.Time, debug bool, msg string) { + f.out(stamp, debug, msg) +} + +func (f funcOut) Close() error { + return f.close() +} + +func FuncOutput(f func(time.Time, bool, string), close func() error) Output { + return funcOut{f, close} +} + +type NopOutput struct{} + +func (NopOutput) Write(time.Time, bool, string) {} + +func (NopOutput) Close() error { return nil } diff --git a/framework/log/syslog.go b/framework/log/syslog.go new file mode 100644 index 0000000..d608f55 --- /dev/null +++ b/framework/log/syslog.go @@ -0,0 +1,62 @@ +//go:build !windows && !plan9 +// +build !windows,!plan9 + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package log + +import ( + "fmt" + "log/syslog" + "os" + "time" +) + +type syslogOut struct { + w *syslog.Writer +} + +func (s syslogOut) Write(stamp time.Time, debug bool, msg string) { + var err error + if debug { + err = s.w.Debug(msg + "\n") + } else { + err = s.w.Info(msg + "\n") + } + + if err != nil { + fmt.Fprintf(os.Stderr, "!!! Failed to send message to syslog daemon: %v\n", err) + } +} + +func (s syslogOut) Close() error { + return s.w.Close() +} + +// SyslogOutput returns a log.Output implementation that will send +// messages to the system syslog daemon. +// +// Regular messages will be written with INFO priority, +// debug messages will be written with DEBUG priority. +// +// Returned log.Output object is goroutine-safe. +func SyslogOutput() (Output, error) { + w, err := syslog.New(syslog.LOG_MAIL|syslog.LOG_INFO, "maddy") + return syslogOut{w}, err +} diff --git a/framework/log/syslog_stub.go b/framework/log/syslog_stub.go new file mode 100644 index 0000000..bc48616 --- /dev/null +++ b/framework/log/syslog_stub.go @@ -0,0 +1,37 @@ +//go:build windows || plan9 +// +build windows plan9 + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package log + +import ( + "errors" +) + +// SyslogOutput returns a log.Output implementation that will send +// messages to the system syslog daemon. +// +// Regular messages will be written with INFO priority, +// debug messages will be written with DEBUG priority. +// +// Returned log.Output object is goroutine-safe. +func SyslogOutput() (Output, error) { + return nil, errors.New("log: syslog output is not supported on windows") +} diff --git a/framework/log/writer.go b/framework/log/writer.go new file mode 100644 index 0000000..1528c2e --- /dev/null +++ b/framework/log/writer.go @@ -0,0 +1,95 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package log + +import ( + "fmt" + "io" + "os" + "strings" + "time" +) + +type wcOutput struct { + timestamps bool + wc io.WriteCloser +} + +func (w wcOutput) Write(stamp time.Time, debug bool, msg string) { + builder := strings.Builder{} + if w.timestamps { + builder.WriteString(stamp.UTC().Format("2006-01-02T15:04:05.000Z ")) + } + if debug { + builder.WriteString("[debug] ") + } + builder.WriteString(msg) + builder.WriteRune('\n') + if _, err := io.WriteString(w.wc, builder.String()); err != nil { + fmt.Fprintf(os.Stderr, "!!! Failed to write message to log: %v\n", err) + } +} + +func (w wcOutput) Close() error { + return w.wc.Close() +} + +// WriteCloserOutput returns a log.Output implementation that +// will write formatted messages to the provided io.Writer. +// +// Closing returned log.Output object will close the underlying +// io.WriteCloser. +// +// Written messages will include timestamp formatted with millisecond +// precision and [debug] prefix for debug messages. +// If timestamps argument is false, timestamps will not be added. +// +// Returned log.Output does not provide its own serialization +// so goroutine-safety depends on the io.Writer. Most operating +// systems have atomic (read: thread-safe) implementations for +// stream I/O, so it should be safe to use WriterOutput with os.File. +func WriteCloserOutput(wc io.WriteCloser, timestamps bool) Output { + return wcOutput{timestamps, wc} +} + +type nopCloser struct { + io.Writer +} + +func (nc nopCloser) Close() error { + return nil +} + +// WriterOutput returns a log.Output implementation that +// will write formatted messages to the provided io.Writer. +// +// Closing returned log.Output object will have no effect on the +// underlying io.Writer. +// +// Written messages will include timestamp formatted with millisecond +// precision and [debug] prefix for debug messages. +// If timestamps argument is false, timestamps will not be added. +// +// Returned log.Output does not provide its own serialization +// so goroutine-safety depends on the io.Writer. Most operating +// systems have atomic (read: thread-safe) implementations for +// stream I/O, so it should be safe to use WriterOutput with os.File. +func WriterOutput(w io.Writer, timestamps bool) Output { + return wcOutput{timestamps, nopCloser{os.Stderr}} +} diff --git a/framework/log/zap.go b/framework/log/zap.go new file mode 100644 index 0000000..23821f8 --- /dev/null +++ b/framework/log/zap.go @@ -0,0 +1,57 @@ +package log + +import ( + "go.uber.org/zap/zapcore" +) + +// TODO: Migrate to using actual zapcore to improve logging performance + +type zapLogger struct { + L Logger +} + +func (l zapLogger) Enabled(level zapcore.Level) bool { + if l.L.Debug { + return true + } + return level > zapcore.DebugLevel +} + +func (l zapLogger) With(fields []zapcore.Field) zapcore.Core { + enc := zapcore.NewMapObjectEncoder() + for _, f := range fields { + f.AddTo(enc) + } + newF := make(map[string]interface{}, len(l.L.Fields)+len(enc.Fields)) + for k, v := range l.L.Fields { + newF[k] = v + } + for k, v := range enc.Fields { + newF[k] = v + } + l.L.Fields = newF + return l +} + +func (l zapLogger) Check(entry zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if l.Enabled(entry.Level) { + return ce.AddCore(entry, l) + } + return ce +} + +func (l zapLogger) Write(entry zapcore.Entry, fields []zapcore.Field) error { + enc := zapcore.NewMapObjectEncoder() + for _, f := range fields { + f.AddTo(enc) + } + if entry.LoggerName != "" { + l.L.Name += "/" + entry.LoggerName + } + l.L.log(entry.Level == zapcore.DebugLevel, l.L.formatMsg(entry.Message, enc.Fields)) + return nil +} + +func (zapLogger) Sync() error { + return nil +} diff --git a/framework/logparser/parse.go b/framework/logparser/parse.go new file mode 100644 index 0000000..a0fac77 --- /dev/null +++ b/framework/logparser/parse.go @@ -0,0 +1,124 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package parser provides utilities for parsing of structured log messsages +// generated by maddy. +package parser + +import ( + "encoding/json" + "strings" + "time" + "unicode" +) + +type ( + Msg struct { + Stamp time.Time + Debug bool + Module string + Message string + Context map[string]interface{} + } + + MalformedMsg struct { + Desc string + Err error + } +) + +const ( + ISO8601_UTC = "2006-01-02T15:04:05.000Z" +) + +func (m MalformedMsg) Error() string { + if m.Err != nil { + return "parse: " + m.Desc + ": " + m.Err.Error() + } + return "parse: " + m.Desc +} + +// Parse parses the message from the maddy log file. +// +// It assumes standard file output, including the [debug] tag and +// ISO 8601 timestamp at the start of each line. Timestamp is assumed to be in +// the UTC, as it is enforced by maddy. +// +// JSON context values are unmarshalled without any additional processing, +// notably that means that all numbers are represented as float64. +func Parse(line string) (Msg, error) { + parts := strings.Split(line, "\t") + if len(parts) != 2 { + // All messages even without a Context have a trailing \t, + // so this one is obviously malformed. + return Msg{}, MalformedMsg{Desc: "missing a tab separator"} + } + + m := Msg{ + Context: map[string]interface{}{}, + } + + // After that, the second part is the context. It can be empty, so don't fail + // if there is none. + if len(parts[1]) != 0 { + if err := json.Unmarshal([]byte(parts[1]), &m.Context); err != nil { + return Msg{}, MalformedMsg{Desc: "context unmarshal", Err: err} + } + } + + // Okay, the first one might contain the timestamp at start. + // Cut it away. + msgParts := strings.SplitN(parts[0], " ", 2) + if len(msgParts) == 1 { + return Msg{}, MalformedMsg{Desc: "missing a timestamp"} + } + + var err error + m.Stamp, err = time.ParseInLocation(ISO8601_UTC, msgParts[0], time.UTC) + if err != nil { + return Msg{}, MalformedMsg{Desc: "timestamp parse", Err: err} + } + + msgText := msgParts[1] + if strings.HasPrefix(msgText, "[debug] ") { + msgText = strings.TrimPrefix(msgText, "[debug] ") + m.Debug = true + } + + moduleText := strings.SplitN(msgText, ": ", 2) + if len(moduleText) == 1 { + // No module prefix, that's fine. + m.Message = msgText + return m, nil + } + + for _, ch := range moduleText[0] { + switch { + case unicode.IsDigit(ch), unicode.IsLetter(ch), ch == '/': + default: + // This is not a module prefix, don't treat it as such. + m.Message = msgText + return m, nil + } + } + + m.Module = moduleText[0] + m.Message = moduleText[1] + + return m, nil +} diff --git a/framework/logparser/parse_test.go b/framework/logparser/parse_test.go new file mode 100644 index 0000000..6361541 --- /dev/null +++ b/framework/logparser/parse_test.go @@ -0,0 +1,113 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package parser + +import ( + "reflect" + "testing" + "time" +) + +func TestParse(t *testing.T) { + test := func(line string, msg Msg, errDesc string) { + t.Helper() + + parsed, err := Parse(line) + if errDesc != "" { + if err == nil { + t.Errorf("Expected an error, got none") + return + } + if err.(MalformedMsg).Desc != errDesc { + t.Errorf("Wrong error desc returned: %v", err.(MalformedMsg).Desc) + return + } + } + if errDesc == "" && err != nil { + t.Errorf("Unexpected error: %v", err) + return + } + + if !reflect.DeepEqual(parsed, msg) { + t.Errorf("Wrong Parse result,\n got %#+v\n want %#+v", parsed, msg) + } + } + + test("2006-01-02T15:04:05.000Z module: hello\t", Msg{ + Stamp: time.Date(2006, time.January, 2, 15, 4, 5, 0, time.UTC), + Module: "module", + Message: "hello", + Context: map[string]interface{}{}, + }, "") + test("2006-01-02T15:04:05.000Z module: hello: whatever\t", Msg{ + Stamp: time.Date(2006, time.January, 2, 15, 4, 5, 0, time.UTC), + Module: "module", + Message: "hello: whatever", + Context: map[string]interface{}{}, + }, "") + test("2006-01-02T15:04:05.000Z module: hello: whatever\t{\"a\":1}", Msg{ + Stamp: time.Date(2006, time.January, 2, 15, 4, 5, 0, time.UTC), + Module: "module", + Message: "hello: whatever", + Context: map[string]interface{}{ + "a": float64(1), + }, + }, "") + test("2006-01-02T15:04:05.000Z module: hello: whatever\t{\"a\":1,\"b\":\"bbb\"}", Msg{ + Stamp: time.Date(2006, time.January, 2, 15, 4, 5, 0, time.UTC), + Module: "module", + Message: "hello: whatever", + Context: map[string]interface{}{ + "a": float64(1), + "b": "bbb", + }, + }, "") + test("2006-01-02T15:04:05.000Z [debug] module: hello: whatever\t{\"a\":1,\"b\":\"bbb\"}", Msg{ + Stamp: time.Date(2006, time.January, 2, 15, 4, 5, 0, time.UTC), + Debug: true, + Module: "module", + Message: "hello: whatever", + Context: map[string]interface{}{ + "a": float64(1), + "b": "bbb", + }, + }, "") + test("2006-01-02T15:04:05.000Z [debug] oink oink: hello: whatever\t{\"a\":1,\"b\":\"bbb\"}", Msg{ + Stamp: time.Date(2006, time.January, 2, 15, 4, 5, 0, time.UTC), + Debug: true, + Message: "oink oink: hello: whatever", + Context: map[string]interface{}{ + "a": float64(1), + "b": "bbb", + }, + }, "") + test("2006-01-02T15:04:05.000Z [debug] whatever\t{\"a\":1,\"b\":\"bbb\"}", Msg{ + Stamp: time.Date(2006, time.January, 2, 15, 4, 5, 0, time.UTC), + Debug: true, + Message: "whatever", + Context: map[string]interface{}{ + "a": float64(1), + "b": "bbb", + }, + }, "") + test("module: hello\t", Msg{}, "timestamp parse") + test("hello\t", Msg{}, "missing a timestamp") + test("2006-01-02T15:04:05.000Z module: hello", Msg{}, "missing a tab separator") + test("2006-01-02T15:04:05.000Z [BROKEN FORMATTING: json: wtf lol omg]: hello map[stringasdasd]", Msg{}, "missing a tab separator") +} diff --git a/framework/module/auth.go b/framework/module/auth.go new file mode 100644 index 0000000..6e8d089 --- /dev/null +++ b/framework/module/auth.go @@ -0,0 +1,44 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import "errors" + +// ErrUnknownCredentials should be returned by auth. provider if supplied +// credentials are valid for it but are not recognized (e.g. not found in +// used DB). +var ErrUnknownCredentials = errors.New("unknown credentials") + +// PlainAuth is the interface implemented by modules providing authentication using +// username:password pairs. +// +// Modules implementing this interface should be registered with "auth." prefix in name. +type PlainAuth interface { + AuthPlain(username, password string) error +} + +// PlainUserDB is a local credentials store that can be managed using maddy command +// utility. +type PlainUserDB interface { + PlainAuth + ListUsers() ([]string, error) + CreateUser(username, password string) error + SetUserPassword(username, password string) error + DeleteUser(username string) error +} diff --git a/framework/module/blob_store.go b/framework/module/blob_store.go new file mode 100644 index 0000000..426a854 --- /dev/null +++ b/framework/module/blob_store.go @@ -0,0 +1,46 @@ +package module + +import ( + "context" + "errors" + "io" +) + +type Blob interface { + Sync() error + io.Writer + io.Closer +} + +var ErrNoSuchBlob = errors.New("blob_store: no such object") + +const UnknownBlobSize int64 = -1 + +// BlobStore is the interface used by modules providing large binary object +// storage. +type BlobStore interface { + // Create creates a new blob for writing. + // + // Sync will be called on the returned Blob object after -all- data has + // been successfully written. + // + // Close without Sync can be assumed to happen due to an unrelated error + // and stored data can be discarded. + // + // blobSize indicates the exact amount of bytes that will be written + // If -1 is passed - it is unknown and implementation will not make + // any assumptions about the blob size. Error can be returned by any + // Blob method if more than than blobSize bytes get written. + // + // Passed context will cover the entire blob write operation. + Create(ctx context.Context, key string, blobSize int64) (Blob, error) + + // Open returns the reader for the object specified by + // passed key. + // + // If no such object exists - ErrNoSuchBlob is returned. + Open(ctx context.Context, key string) (io.ReadCloser, error) + + // Delete removes a set of keys from store. Non-existent keys are ignored. + Delete(ctx context.Context, keys []string) error +} diff --git a/framework/module/check.go b/framework/module/check.go new file mode 100644 index 0000000..4802f6e --- /dev/null +++ b/framework/module/check.go @@ -0,0 +1,113 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/foxcpp/maddy/framework/buffer" +) + +// Check is the module interface that is meant for read-only (with the +// exception of the message header modifications) (meta-)data checking. +// +// Modules implementing this interface should be registered with "check." +// prefix in name. +type Check interface { + // CheckStateForMsg initializes the "internal" check state required for + // processing of the new message. + // + // NOTE: Returned CheckState object must be hashable (usable as a map key). + // This is used to deduplicate Check* calls, the easiest way to achieve + // this is to have CheckState as a pointer to some struct, all pointers + // are hashable. + CheckStateForMsg(ctx context.Context, msgMeta *MsgMetadata) (CheckState, error) +} + +// EarlyCheck is an optional module interface that can be implemented +// by module implementing Check. +// +// It is used as an optimization to reject obviously malicious connections +// before allocating resources for SMTP session. +// +// The Status of this check is accept (no error) or reject (error) only, no +// advanced handling is available (such as 'quarantine' action and headers +// prepending). +// +// If it s necessary to defer or affect further message processing +// without outright killing the session, ConnState.ModData can be +// used to store necessary information. +// +// It may be called multiple times for the same connection if TLS is negotiated +// via STARTTLS. In this case, no state will be passed between before-TLS +// context to the TLS one. +type EarlyCheck interface { + CheckConnection(ctx context.Context, state *ConnState) error +} + +type CheckState interface { + // CheckConnection is executed once when client sends a new message. + CheckConnection(ctx context.Context) CheckResult + + // CheckSender is executed once when client sends the message sender + // information (e.g. on the MAIL FROM command). + CheckSender(ctx context.Context, mailFrom string) CheckResult + + // CheckRcpt is executed for each recipient when its address is received + // from the client (e.g. on the RCPT TO command). + CheckRcpt(ctx context.Context, rcptTo string) CheckResult + + // CheckBody is executed once after the message body is received and + // buffered in memory or on disk. + // + // Check code should use passed mutex when working with the message header. + // Body can be read without locking it since it is read-only. + CheckBody(ctx context.Context, header textproto.Header, body buffer.Buffer) CheckResult + + // Close is called after the message processing ends, even if any of the + // Check* functions return an error. + Close() error +} + +type CheckResult struct { + // Reason is the error that is reported to the message source + // if check decided that the message should be rejected. + Reason error + + // Reject is the flag that specifies that the message + // should be rejected. + Reject bool + + // Quarantine is the flag that specifies that the message + // is considered "possibly malicious" and should be + // put into Junk mailbox. + // + // This value is copied into MsgMetadata by the msgpipeline. + Quarantine bool + + // AuthResult is the information that is supposed to + // be included in Authentication-Results header. + AuthResult []authres.Result + + // Header is the header fields that should be + // added to the header after all checks. + Header textproto.Header +} diff --git a/framework/module/delivery_target.go b/framework/module/delivery_target.go new file mode 100644 index 0000000..9a7c1e0 --- /dev/null +++ b/framework/module/delivery_target.go @@ -0,0 +1,93 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" +) + +// DeliveryTarget interface represents abstract storage for the message data +// (typically persistent) or other kind of component that can be used as a +// final destination for the message. +// +// Modules implementing this interface should be registered with "target." +// prefix in name. +type DeliveryTarget interface { + // Start starts the delivery of a new message. + // + // The domain part of the MAIL FROM address is assumed to be U-labels with + // NFC normalization and case-folding applied. The message source should + // ensure that by calling address.CleanDomain if necessary. + Start(ctx context.Context, msgMeta *MsgMetadata, mailFrom string) (Delivery, error) +} + +type Delivery interface { + // AddRcpt adds the target address for the message. + // + // The domain part of the address is assumed to be U-labels with NFC normalization + // and case-folding applied. The message source should ensure that by + // calling address.CleanDomain if necessary. + // + // Implementation should assume that no case-folding or deduplication was + // done by caller code. Its implementation responsibility to do so if it is + // necessary. It is not recommended to reject duplicated recipients, + // however. They should be silently ignored. + // + // Implementation should do as much checks as possible here and reject + // recipients that can't be used. Note: MsgMetadata object passed to Start + // contains BodyLength field. If it is non-zero, it can be used to check + // storage quota for the user before Body. + AddRcpt(ctx context.Context, rcptTo string, opts smtp.RcptOptions) error + + // Body sets the body and header contents for the message. + // If this method fails, message is assumed to be undeliverable + // to all recipients. + // + // Implementation should avoid doing any persistent changes to the + // underlying storage until Commit is called. If that is not possible, + // Abort should (attempt to) rollback any such changes. + // + // If Body can't be implemented without per-recipient failures, + // then delivery object should also implement PartialDelivery interface + // for use by message sources that are able to make sense of per-recipient + // errors. + // + // Here is the example of possible implementation for maildir-based + // storage: + // Calling Body creates a file in tmp/ directory. + // Commit moves the created file to new/ directory. + // Abort removes the created file. + Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error + + // Abort cancels message delivery. + // + // All changes made to the underlying storage should be aborted at this + // point, if possible. + Abort(ctx context.Context) error + + // Commit completes message delivery. + // + // It generally should never fail, since failures here jeopardize + // atomicity of the delivery if multiple targets are used. + Commit(ctx context.Context) error +} diff --git a/framework/module/dummy.go b/framework/module/dummy.go new file mode 100644 index 0000000..0930d4d --- /dev/null +++ b/framework/module/dummy.go @@ -0,0 +1,87 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" +) + +// Dummy is a struct that implements PlainAuth and DeliveryTarget +// interfaces but does nothing. Useful for testing. +// +// It is always registered under the 'dummy' name and can be used in both tests +// and the actual server code (but the latter is kinda pointless). +type Dummy struct{ instName string } + +func (d *Dummy) AuthPlain(username, _ string) error { + return nil +} + +func (d *Dummy) Lookup(_ context.Context, _ string) (string, bool, error) { + return "", false, nil +} + +func (d *Dummy) LookupMulti(_ context.Context, _ string) ([]string, error) { + return []string{""}, nil +} + +func (d *Dummy) Name() string { + return "dummy" +} + +func (d *Dummy) InstanceName() string { + return d.instName +} + +func (d *Dummy) Init(_ *config.Map) error { + return nil +} + +func (d *Dummy) Start(ctx context.Context, msgMeta *MsgMetadata, mailFrom string) (Delivery, error) { + return dummyDelivery{}, nil +} + +type dummyDelivery struct{} + +func (dd dummyDelivery) AddRcpt(ctx context.Context, rcptTo string, opts smtp.RcptOptions) error { + return nil +} + +func (dd dummyDelivery) Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error { + return nil +} + +func (dd dummyDelivery) Abort(ctx context.Context) error { + return nil +} + +func (dd dummyDelivery) Commit(ctx context.Context) error { + return nil +} + +func init() { + Register("dummy", func(_, instName string, _, _ []string) (Module, error) { + return &Dummy{instName: instName}, nil + }) +} diff --git a/framework/module/imap_filter.go b/framework/module/imap_filter.go new file mode 100644 index 0000000..6b0fd46 --- /dev/null +++ b/framework/module/imap_filter.go @@ -0,0 +1,43 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" +) + +// IMAPFilter is interface used by modules that want to modify IMAP-specific message +// attributes on delivery. +// +// Modules implementing this interface should be registered with namespace prefix +// "imap.filter". +type IMAPFilter interface { + // IMAPFilter is called when message is about to be stored in IMAP-compatible + // storage. It is called only for messages delivered over SMTP, hdr and body + // contain the message exactly how it will be stored. + // + // Filter can change the target directory by returning non-empty folder value. + // Additionally it can add additional IMAP flags to the message by returning + // them. + // + // Errors returned by IMAPFilter will be just logged and will not cause delivery + // to fail. + IMAPFilter(accountName string, rcptTo string, meta *MsgMetadata, hdr textproto.Header, body buffer.Buffer) (folder string, flags []string, err error) +} diff --git a/framework/module/instances.go b/framework/module/instances.go new file mode 100644 index 0000000..aa6f148 --- /dev/null +++ b/framework/module/instances.go @@ -0,0 +1,105 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "fmt" + "io" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/log" +) + +var ( + instances = make(map[string]struct { + mod Module + cfg *config.Map + }) + aliases = make(map[string]string) + + Initialized = make(map[string]bool) +) + +// RegisterInstance adds module instance to the global registry. +// +// Instance name must be unique. Second RegisterInstance with same instance +// name will replace previous. +func RegisterInstance(inst Module, cfg *config.Map) { + instances[inst.InstanceName()] = struct { + mod Module + cfg *config.Map + }{inst, cfg} +} + +// RegisterAlias creates an association between a certain name and instance name. +// +// After RegisterAlias, module.GetInstance(aliasName) will return the same +// result as module.GetInstance(instName). +func RegisterAlias(aliasName, instName string) { + aliases[aliasName] = instName +} + +func HasInstance(name string) bool { + aliasedName := aliases[name] + if aliasedName != "" { + name = aliasedName + } + + _, ok := instances[name] + return ok +} + +// GetInstance returns module instance from global registry, initializing it if +// necessary. +// +// Error is returned if module initialization fails or module instance does not +// exists. +func GetInstance(name string) (Module, error) { + aliasedName := aliases[name] + if aliasedName != "" { + name = aliasedName + } + + mod, ok := instances[name] + if !ok { + return nil, fmt.Errorf("unknown config block: %s", name) + } + + // Break circular dependencies. + if Initialized[name] { + return mod.mod, nil + } + + Initialized[name] = true + if err := mod.mod.Init(mod.cfg); err != nil { + return mod.mod, err + } + + if closer, ok := mod.mod.(io.Closer); ok { + hooks.AddHook(hooks.EventShutdown, func() { + log.Debugf("close %s (%s)", mod.mod.Name(), mod.mod.InstanceName()) + if err := closer.Close(); err != nil { + log.Printf("module %s (%s) close failed: %v", mod.mod.Name(), mod.mod.InstanceName(), err) + } + }) + } + + return mod.mod, nil +} diff --git a/framework/module/modifier.go b/framework/module/modifier.go new file mode 100644 index 0000000..2a5b10f --- /dev/null +++ b/framework/module/modifier.go @@ -0,0 +1,83 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" +) + +// Modifier is the module interface for modules that can mutate the +// processed message or its meta-data. +// +// Currently, the message body can't be mutated for efficiency and +// correctness reasons: It would require "rebuffering" (see buffer.Buffer doc), +// can invalidate assertions made on the body contents before modification and +// will break DKIM signatures. +// +// Only message header can be modified. Furthermore, it is highly discouraged for +// modifiers to remove or change existing fields to prevent issues outlined +// above. +// +// Calls on ModifierState are always strictly ordered. +// RewriteRcpt is newer called before RewriteSender and RewriteBody is never called +// before RewriteRcpts. This allows modificator code to save values +// passed to previous calls for use in later operations. +// +// Modules implementing this interface should be registered with "modify." prefix in name. +type Modifier interface { + // ModStateForMsg initializes modifier "internal" state + // required for processing of the message. + ModStateForMsg(ctx context.Context, msgMeta *MsgMetadata) (ModifierState, error) +} + +type ModifierState interface { + // RewriteSender allows modifier to replace MAIL FROM value. + // If no changes are required, this method returns its + // argument, otherwise it returns a new value. + // + // Note that per-source/per-destination modifiers are executed + // after routing decision is made so changed value will have no + // effect on it. + // + // Also note that MsgMeta.OriginalFrom will still contain the original value + // for purposes of tracing. It should not be modified by this method. + RewriteSender(ctx context.Context, mailFrom string) (string, error) + + // RewriteRcpt replaces RCPT TO value. + // If no changed are required, this method returns its argument as slice, + // otherwise it returns a slice with 1 or more new values. + // + // MsgPipeline will take of populating MsgMeta.OriginalRcpts. RewriteRcpt + // doesn't do it. + RewriteRcpt(ctx context.Context, rcptTo string) ([]string, error) + + // RewriteBody modifies passed Header argument and may optionally + // inspect the passed body buffer to make a decision on new header field values. + // + // There is no way to modify the body and RewriteBody should avoid + // removing existing header fields and changing their values. + RewriteBody(ctx context.Context, h *textproto.Header, body buffer.Buffer) error + + // Close is called after the message processing ends, even if any of the + // Rewrite* functions return an error. + Close() error +} diff --git a/framework/module/module.go b/framework/module/module.go new file mode 100644 index 0000000..2dbb45e --- /dev/null +++ b/framework/module/module.go @@ -0,0 +1,90 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package module contains modules registry and interfaces implemented +// by modules. +// +// Interfaces are placed here to prevent circular dependencies. +// +// Each interface required by maddy for operation is provided by some object +// called "module". This includes authentication, storage backends, DKIM, +// email filters, etc. Each module may serve multiple functions. I.e. it can +// be IMAP storage backend, SMTP downstream and authentication provider at the +// same moment. +// +// Each module gets its own unique name (sql for go-imap-sql, proxy for +// proxy module, local for local delivery perhaps, etc). Each module instance +// also can have its own unique name can be used to refer to it in +// configuration. +package module + +import ( + "github.com/foxcpp/maddy/framework/config" +) + +// Module is the interface implemented by all maddy module instances. +// +// It defines basic methods used to identify instances. +// +// Additionally, module can implement io.Closer if it needs to perform clean-up +// on shutdown. If module starts long-lived goroutines - they should be stopped +// *before* Close method returns to ensure graceful shutdown. +type Module interface { + // Init performs actual initialization of the module. + // + // It is not done in FuncNewModule so all module instances are + // registered at time of initialization, thus initialization does not + // depends on ordering of configuration blocks and modules can reference + // each other without any problems. + // + // Module can use passed config.Map to read its configuration variables. + Init(*config.Map) error + + // Name method reports module name. + // + // It is used to reference module in the configuration and in logs. + Name() string + + // InstanceName method reports unique name of this module instance or empty + // string if module instance is unnamed. + InstanceName() string +} + +// FuncNewModule is function that creates new instance of module with specified name. +// +// Module.InstanceName() of the returned module object should return instName. +// aliases slice contains other names that can be used to reference created +// module instance. +// +// If module is defined inline, instName will be empty and all values +// specified after module name in configuration will be in inlineArgs. +type FuncNewModule func(modName, instName string, aliases, inlineArgs []string) (Module, error) + +// FuncNewEndpoint is a function that creates new instance of endpoint +// module. +// +// Compared to regular modules, endpoint module instances are: +// - Not registered in the global registry. +// - Can't be defined inline. +// - Don't have an unique name +// - All config arguments are always passed as an 'addrs' slice and not used as +// names. +// +// As a consequence of having no per-instance name, InstanceName of the module +// object always returns the same value as Name. +type FuncNewEndpoint func(modName string, addrs []string) (Module, error) diff --git a/framework/module/module_specific_data.go b/framework/module/module_specific_data.go new file mode 100644 index 0000000..4155a61 --- /dev/null +++ b/framework/module/module_specific_data.go @@ -0,0 +1,63 @@ +package module + +import ( + "encoding/json" + "fmt" + "sync" +) + +// ModSpecificData is a container that allows modules to attach +// additional context data to framework objects such as SMTP connections +// without conflicting with each other and ensuring each module +// gets its own namespace. +// +// It must not be used to store stateful objects that may need +// a specific cleanup routine as ModSpecificData does not provide +// any lifetime management. +// +// Stored data must be serializable to JSON for state persistence +// e.g. when message is stored in a on-disk queue. +type ModSpecificData struct { + modDataLck sync.RWMutex + modData map[string]interface{} +} + +func (msd *ModSpecificData) modKey(m Module, perInstance bool) string { + if !perInstance { + return m.Name() + } + instName := m.InstanceName() + if instName == "" { + instName = fmt.Sprintf("%x", m) + } + return m.Name() + "/" + instName +} + +func (msd *ModSpecificData) MarshalJSON() ([]byte, error) { + msd.modDataLck.RLock() + defer msd.modDataLck.RUnlock() + return json.Marshal(msd.modData) +} + +func (msd *ModSpecificData) UnmarshalJSON(b []byte) error { + msd.modDataLck.Lock() + defer msd.modDataLck.Unlock() + return json.Unmarshal(b, &msd.modData) +} + +func (msd *ModSpecificData) Set(m Module, perInstance bool, value interface{}) { + key := msd.modKey(m, perInstance) + msd.modDataLck.Lock() + defer msd.modDataLck.Unlock() + if msd.modData == nil { + msd.modData = make(map[string]interface{}) + } + msd.modData[key] = value +} + +func (msd *ModSpecificData) Get(m Module, perInstance bool) interface{} { + key := msd.modKey(m, perInstance) + msd.modDataLck.RLock() + defer msd.modDataLck.RUnlock() + return msd.modData[key] +} diff --git a/framework/module/msgmetadata.go b/framework/module/msgmetadata.go new file mode 100644 index 0000000..bfbe634 --- /dev/null +++ b/framework/module/msgmetadata.go @@ -0,0 +1,158 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "crypto/rand" + "crypto/tls" + "encoding/hex" + "io" + "net" + + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/future" +) + +// ConnState structure holds the state information of the protocol used to +// accept this message. +type ConnState struct { + // IANA name (ESMTP, ESMTPS, etc) of the protocol message was received + // over. If the message was generated locally, this field is empty. + Proto string + + // Information about the SMTP connection, including HELO hostname and + // source IP. Valid only if Proto refers the SMTP protocol or its variant + // (e.g. LMTP). + Hostname string + LocalAddr net.Addr + RemoteAddr net.Addr + TLS tls.ConnectionState + + // The RDNSName field contains the result of Reverse DNS lookup on the + // client IP. + // + // The underlying type is the string or untyped nil value. It is the + // message source responsibility to populate this field. + // + // Valid values of this field consumers need to be aware of: + // RDNSName = nil + // The reverse DNS lookup is not applicable for that message source. + // Typically the case for messages generated locally. + // RDNSName != nil, but Get returns nil + // The reverse DNS lookup was attempted, but resulted in an error. + // Consumers should assume that the PTR record doesn't exist. + RDNSName *future.Future + + // If the client successfully authenticated using a username/password pair. + // This field contains the username. + AuthUser string + + // If the client successfully authenticated using a username/password pair. + // This field should be cleaned if the ConnState object is serialized + AuthPassword string + + ModData ModSpecificData +} + +// MsgMetadata structure contains all information about the origin of +// the message and all associated flags indicating how it should be handled +// by components. +// +// All fields should be considered read-only except when otherwise is noted. +// Module instances should avoid keeping reference to the instance passed to it +// and copy the structure using DeepCopy method instead. +// +// Compatibility with older values should be considered when changing this +// structure since it is serialized to the disk by the queue module using +// JSON. Modules should correctly handle missing or invalid values. +type MsgMetadata struct { + // Unique identifier for this message. Randomly generated by the + // message source module. + ID string + + // Original message sender address as it was received by the message source. + // + // Note that this field is meant for use for tracing purposes. + // All routing and other decisions should be made based on the sender address + // passed separately (for example, mailFrom argument for CheckSender function) + // Note that addresses may contain unescaped Unicode characters. + OriginalFrom string + + // If set - no SrcHostname and SrcAddr will be added to Received + // header. These fields are still written to the server log. + DontTraceSender bool + + // Quarantine is a message flag that is should be set if message is + // considered "suspicious" and should be put into "Junk" folder + // in the storage. + // + // This field should not be modified by the checks that verify + // the message. It is set only by the message pipeline. + Quarantine bool + + // OriginalRcpts contains the mapping from the final recipient to the + // recipient that was presented by the client. + // + // MsgPipeline will update that field when recipient modifiers + // are executed. + // + // It should be used when reporting information back to client (via DSN, + // for example) to prevent disclosing information about aliases + // which is usually unwanted. + OriginalRcpts map[string]string + + // SMTPOpts contains the SMTP MAIL FROM command arguments, if the message + // was accepted over SMTP or SMTP-like protocol (such as LMTP). + // + // Note that the Size field should not be used as source of information about + // the body size. Especially since it counts the header too whereas + // Buffer.Len does not. + SMTPOpts smtp.MailOptions + + // Conn contains the information about the underlying protocol connection + // that was used to accept this message. The referenced instance may be shared + // between multiple messages. + // + // It can be nil for locally generated messages. + Conn *ConnState + + // This is set by endpoint/smtp to indicate that body contains "TLS-Required: No" + // header. It is only meaningful if server has seen the body at least once + // (e.g. the message was passed via queue). + TLSRequireOverride bool +} + +// DeepCopy creates a copy of the MsgMetadata structure, also +// copying contents of the maps and slices. +// +// There are a few exceptions, however: +// - SrcAddr is not copied and copy field references original value. +func (msgMeta *MsgMetadata) DeepCopy() *MsgMetadata { + cpy := *msgMeta + // There is no good way to copy net.Addr, but it should not be + // modified by anything anyway so we are safe. + return &cpy +} + +// GenerateMsgID generates a string usable as MsgID field in module.MsgMeta. +func GenerateMsgID() (string, error) { + rawID := make([]byte, 4) + _, err := io.ReadFull(rand.Reader, rawID) + return hex.EncodeToString(rawID), err +} diff --git a/framework/module/mxauth.go b/framework/module/mxauth.go new file mode 100644 index 0000000..5226fb0 --- /dev/null +++ b/framework/module/mxauth.go @@ -0,0 +1,156 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "context" + "crypto/tls" +) + +const ( + AuthDisabled = "off" + AuthMTASTS = "mtasts" + AuthDNSSEC = "dnssec" + AuthCommonDomain = "common_domain" +) + +type ( + TLSLevel int + MXLevel int +) + +const ( + TLSNone TLSLevel = iota + TLSEncrypted + TLSAuthenticated +) + +const ( + MXNone MXLevel = iota + MX_MTASTS + MX_DNSSEC +) + +func (l TLSLevel) String() string { + switch l { + case TLSNone: + return "none" + case TLSEncrypted: + return "encrypted" + case TLSAuthenticated: + return "authenticated" + } + return "???" +} + +func (l MXLevel) String() string { + switch l { + case MXNone: + return "none" + case MX_MTASTS: + return "mtasts" + case MX_DNSSEC: + return "dnssec" + } + return "???" +} + +type ( + // MXAuthPolicy is an object that provides security check for outbound connections. + // It can do one of the following: + // + // - Check effective TLS level or MX level against some configured or + // discovered value. + // E.g. local policy. + // + // - Raise the security level if certain condition about used MX or + // connection is met. + // E.g. DANE MXAuthPolicy raises TLS level to Authenticated if a matching + // TLSA record is discovered. + // + // - Reject the connection if certain condition about used MX or + // connection is _not_ met. + // E.g. An enforced MTA-STS MXAuthPolicy rejects MX records not matching it. + // + // It is not recommended to mix different types of behavior described above + // in the same implementation. + // Specifically, the first type is used mostly for local policies and is not + // really practical. + // + // Modules implementing this interface should be registered with "mx_auth." + // prefix in name. + MXAuthPolicy interface { + Start(*MsgMetadata) DeliveryMXAuthPolicy + + // Weight is an integer in range 0-1000 that represents relative + // ordering of policy application. + Weight() int + } + + // DeliveryMXAuthPolicy is an interface of per-delivery object that estabilishes + // and verifies required and effective security for MX records and TLS + // connections. + DeliveryMXAuthPolicy interface { + // PrepareDomain is called before DNS MX lookup and may asynchronously + // start additional lookups necessary for policy application in CheckMX + // or CheckConn. + // + // If there any errors - they should be deferred to the CheckMX or + // CheckConn call. + PrepareDomain(ctx context.Context, domain string) + + // PrepareConn is called before connection and may asynchronously + // start additional lookups necessary for policy application in + // CheckConn. + // + // If there are any errors - they should be deferred to the CheckConn + // call. + PrepareConn(ctx context.Context, mx string) + + // CheckMX is called to check whether the policy permits to use a MX. + // + // mxLevel contains the MX security level estabilished by checks + // executed before. + // + // domain is passed to the CheckMX to allow simpler implementation + // of stateless policy objects. + // + // dnssec is true if the MX lookup was performed using DNSSEC-enabled + // resolver and the zone is signed and its signature is valid. + CheckMX(ctx context.Context, mxLevel MXLevel, domain, mx string, dnssec bool) (MXLevel, error) + + // CheckConn is called to check whether the policy permits to use this + // connection. + // + // tlsLevel and mxLevel contain the TLS security level estabilished by + // checks executed before. + // + // domain is passed to the CheckConn to allow simpler implementation + // of stateless policy objects. + // + // If tlsState.HandshakeCompleted is false, TLS is not used. If + // tlsState.VerifiedChains is nil, InsecureSkipVerify was used (no + // ServerName or PKI check was done). + CheckConn(ctx context.Context, mxLevel MXLevel, tlsLevel TLSLevel, domain, mx string, tlsState tls.ConnectionState) (TLSLevel, error) + + // Reset cleans the internal object state for use with another message. + // newMsg may be nil if object is not needed anymore. + Reset(newMsg *MsgMetadata) + } +) diff --git a/framework/module/partial_delivery.go b/framework/module/partial_delivery.go new file mode 100644 index 0000000..beeb46e --- /dev/null +++ b/framework/module/partial_delivery.go @@ -0,0 +1,58 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" +) + +// StatusCollector is an object that is passed by message source +// that is interested in intermediate status reports about partial +// delivery failures. +type StatusCollector interface { + // SetStatus sets the error associated with the recipient. + // + // rcptTo should match exactly the value that was passed to the + // AddRcpt, i.e. if any translations was made by the target, + // they should not affect the rcptTo argument here. + // + // It should not be called multiple times for the same + // value of rcptTo. It also should not be called + // after BodyNonAtomic returns. + // + // SetStatus is goroutine-safe. Implementations + // provide necessary serialization. + SetStatus(rcptTo string, err error) +} + +// PartialDelivery is an optional interface that may be implemented +// by the object returned by DeliveryTarget.Start. See PartialDelivery.BodyNonAtomic +// documentation for details. +type PartialDelivery interface { + // BodyNonAtomic is similar to Body method of the regular Delivery interface + // with the except that it allows target to reject the body only for some + // recipients by setting statuses using passed collector object. + // + // This interface is preferred by the LMTP endpoint and queue implementation + // to ensure correct handling of partial failures. + BodyNonAtomic(ctx context.Context, c StatusCollector, header textproto.Header, body buffer.Buffer) +} diff --git a/framework/module/registry.go b/framework/module/registry.go new file mode 100644 index 0000000..c52210f --- /dev/null +++ b/framework/module/registry.go @@ -0,0 +1,103 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "sync" + + "github.com/foxcpp/maddy/framework/log" +) + +var ( + // NoRun makes sure modules do not start any bacground tests. + // + // If it set - modules should not perform any actual work and should stop + // once the configuration is read and verified to be correct. + // TODO: Replace it with separation of Init and Run at interface level. + NoRun = false + + modules = make(map[string]FuncNewModule) + endpoints = make(map[string]FuncNewEndpoint) + modulesLock sync.RWMutex +) + +// Register adds module factory function to global registry. +// +// name must be unique. Register will panic if module with specified name +// already exists in registry. +// +// You probably want to call this function from func init() of module package. +func Register(name string, factory FuncNewModule) { + modulesLock.Lock() + defer modulesLock.Unlock() + + if _, ok := modules[name]; ok { + panic("Register: module with specified name is already registered: " + name) + } + + modules[name] = factory +} + +// RegisterDeprecated adds module factory function to global registry. +// +// It prints warning to the log about name being deprecated and suggests using +// a new name. +func RegisterDeprecated(name, newName string, factory FuncNewModule) { + Register(name, func(modName, instName string, aliases, inlineArgs []string) (Module, error) { + log.Printf("module initialized via deprecated name %s, %s should be used instead; deprecated name may be removed in the next version", name, newName) + return factory(modName, instName, aliases, inlineArgs) + }) +} + +// Get returns module from global registry. +// +// This function does not return endpoint-type modules, use GetEndpoint for +// that. +// Nil is returned if no module with specified name is registered. +func Get(name string) FuncNewModule { + modulesLock.RLock() + defer modulesLock.RUnlock() + + return modules[name] +} + +// GetEndpoints returns an endpoint module from global registry. +// +// Nil is returned if no module with specified name is registered. +func GetEndpoint(name string) FuncNewEndpoint { + modulesLock.RLock() + defer modulesLock.RUnlock() + + return endpoints[name] +} + +// RegisterEndpoint registers an endpoint module. +// +// See FuncNewEndpoint for information about +// differences of endpoint modules from regular modules. +func RegisterEndpoint(name string, factory FuncNewEndpoint) { + modulesLock.Lock() + defer modulesLock.Unlock() + + if _, ok := endpoints[name]; ok { + panic("Register: module with specified name is already registered: " + name) + } + + endpoints[name] = factory +} diff --git a/framework/module/storage.go b/framework/module/storage.go new file mode 100644 index 0000000..c332569 --- /dev/null +++ b/framework/module/storage.go @@ -0,0 +1,50 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + imapbackend "github.com/emersion/go-imap/backend" +) + +// Storage interface is a slightly modified go-imap's Backend interface +// (authentication is removed). +// +// Modules implementing this interface should be registered with prefix +// "storage." in name. +type Storage interface { + // GetOrCreateIMAPAcct returns User associated with storage account specified by + // the name. + // + // If it doesn't exists - it should be created. + GetOrCreateIMAPAcct(username string) (imapbackend.User, error) + GetIMAPAcct(username string) (imapbackend.User, error) + + // Extensions returns list of IMAP extensions supported by backend. + IMAPExtensions() []string +} + +// ManageableStorage is an extended Storage interface that allows to +// list existing accounts, create and delete them. +type ManageableStorage interface { + Storage + + ListIMAPAccts() ([]string, error) + CreateIMAPAcct(username string) error + DeleteIMAPAcct(username string) error +} diff --git a/framework/module/table.go b/framework/module/table.go new file mode 100644 index 0000000..3e8e19e --- /dev/null +++ b/framework/module/table.go @@ -0,0 +1,43 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import "context" + +// Table is the interface implemented by module that implementation string-to-string +// translation. +// +// Modules implementing this interface should be registered with prefix +// "table." in name. +type Table interface { + Lookup(ctx context.Context, s string) (string, bool, error) +} + +// MultiTable is the interface that module can implement in addition to Table +// if it can provide multiple values as a lookup result. +type MultiTable interface { + LookupMulti(ctx context.Context, s string) ([]string, error) +} + +type MutableTable interface { + Table + Keys() ([]string, error) + RemoveKey(k string) error + SetKey(k, v string) error +} diff --git a/framework/module/tls_loader.go b/framework/module/tls_loader.go new file mode 100644 index 0000000..06184c0 --- /dev/null +++ b/framework/module/tls_loader.go @@ -0,0 +1,41 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package module + +import ( + "crypto/tls" +) + +// TLSLoader interface is module interface that can be used to supply TLS +// certificates to TLS-enabled endpoints. +// +// The interface is intentionally kept simple, all configuration and parameters +// necessary are to be provided using conventional module configuration. +// +// If loader returns multiple certificate chains - endpoint will serve them +// based on SNI matching. +// +// Note that loading function will be called for each connections - it is +// highly recommended to cache parsed form. +// +// Modules implementing this interface should be registered with prefix +// "tls.loader." in name. +type TLSLoader interface { + ConfigureTLS(c *tls.Config) error +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..668bf27 --- /dev/null +++ b/go.mod @@ -0,0 +1,173 @@ +module github.com/foxcpp/maddy + +go 1.23.1 + +toolchain go1.23.5 + +require ( + blitiri.com.ar/go/spf v1.5.1 + github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 + github.com/c0va23/go-proxyprotocol v0.9.1 + github.com/caddyserver/certmagic v0.21.7 + github.com/emersion/go-imap v1.2.2-0.20220928192137-6fac715be9cf + github.com/emersion/go-imap-compress v0.0.0-20201103190257-14809af1d1b9 + github.com/emersion/go-imap-sortthread v1.2.0 + github.com/emersion/go-message v0.18.2 + github.com/emersion/go-milter v0.4.1 + github.com/emersion/go-msgauth v0.6.8 + github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 + github.com/emersion/go-smtp v0.21.3 + github.com/foxcpp/go-dovecot-sasl v0.0.0-20200522223722-c4699d7a24bf + github.com/foxcpp/go-imap-backend-tests v0.0.0-20220105184719-e80aa29a5e16 + github.com/foxcpp/go-imap-i18nlevel v0.0.0-20200208001533-d6ec88553005 + github.com/foxcpp/go-imap-mess v0.0.0-20230108134257-b7ec3a649613 + github.com/foxcpp/go-imap-namespace v0.0.0-20200802091432-08496dd8e0ed + github.com/foxcpp/go-imap-sql v0.5.1-0.20250124140007-8da5567429d5 + github.com/foxcpp/go-mockdns v1.1.0 + github.com/foxcpp/go-mtasts v0.0.0-20240130093538-1438da2e5932 + github.com/go-ldap/ldap/v3 v3.4.10 + github.com/go-sql-driver/mysql v1.8.1 + github.com/google/uuid v1.6.0 + github.com/hashicorp/go-hclog v1.6.3 + github.com/johannesboyne/gofakes3 v0.0.0-20210704111953-6a9f95c2941c + github.com/lib/pq v1.10.9 + github.com/libdns/acmedns v0.2.0 + github.com/libdns/alidns v1.0.3 + github.com/libdns/cloudflare v0.1.1 + github.com/libdns/digitalocean v0.0.0-20230728223659-4f9064657aea + github.com/libdns/gandi v1.0.3 + github.com/libdns/gcore v0.0.0-20250127070537-4a9d185c9d20 + github.com/libdns/googleclouddns v1.1.0 + github.com/libdns/hetzner v0.0.1 + github.com/libdns/leaseweb v0.4.0 + github.com/libdns/libdns v0.2.2 + github.com/libdns/metaname v0.3.0 + github.com/libdns/namecheap v0.0.0-20211109042440-fc7440785c8e + github.com/libdns/namedotcom v0.3.3 + github.com/libdns/rfc2136 v0.1.1 + github.com/libdns/route53 v1.5.1 + github.com/libdns/vultr v1.0.0 + github.com/mattn/go-sqlite3 v1.14.24 + github.com/miekg/dns v1.1.63 + github.com/minio/minio-go/v7 v7.0.84 + github.com/netauth/netauth v0.6.2 + github.com/prometheus/client_golang v1.20.5 + github.com/urfave/cli/v2 v2.27.5 + go.uber.org/zap v1.27.0 + golang.org/x/crypto v0.32.0 + golang.org/x/net v0.34.0 + golang.org/x/sync v0.10.0 + golang.org/x/text v0.21.0 + modernc.org/sqlite v1.34.5 +) + +require ( + cloud.google.com/go/auth v0.14.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect + cloud.google.com/go/compute/metadata v0.6.0 // indirect + filippo.io/edwards25519 v1.1.0 // indirect + github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect + github.com/G-Core/gcore-dns-sdk-go v0.2.9 // indirect + github.com/aws/aws-sdk-go v1.44.40 // indirect + github.com/aws/aws-sdk-go-v2 v1.33.0 // indirect + github.com/aws/aws-sdk-go-v2/config v1.29.1 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.54 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 // indirect + github.com/aws/aws-sdk-go-v2/service/route53 v1.48.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 // indirect + github.com/aws/smithy-go v1.22.2 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/caddyserver/zerossl v0.1.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect + github.com/digitalocean/godo v1.134.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.4 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/googleapis/gax-go/v2 v2.14.1 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/magiconair/properties v1.8.9 // indirect + github.com/mailru/easyjson v0.9.0 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mholt/acmez/v3 v3.0.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/netauth/protocol v0.0.0-20210918062754-7fee492ffcbd // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/pierrec/lz4 v2.6.1+incompatible // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.12.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.19.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/vultr/govultr/v3 v3.14.1 // indirect + github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + github.com/zeebo/blake3 v0.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect + go.opentelemetry.io/otel v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.34.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap/exp v0.3.0 // indirect + golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/oauth2 v0.25.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/time v0.9.0 // indirect + golang.org/x/tools v0.29.0 // indirect + google.golang.org/api v0.218.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 // indirect + google.golang.org/grpc v1.70.0 // indirect + google.golang.org/protobuf v1.36.4 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + gotest.tools v2.2.0+incompatible // indirect + modernc.org/libc v1.61.9 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.8.2 // indirect +) + +replace github.com/emersion/go-imap => github.com/foxcpp/go-imap v1.0.0-beta.1.0.20220623182312-df940c324887 + +replace github.com/emersion/go-smtp => github.com/foxcpp/go-smtp v1.21.4-0.20250124171104-c8519ae4fb23 // v1.21.3+maddy.1 + +replace github.com/libdns/gandi => github.com/foxcpp/libdns-gandi v1.0.4-0.20240127130558-4782f9d5ce3e // v1.0.3+maddy.1 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..4d3eaea --- /dev/null +++ b/go.sum @@ -0,0 +1,1329 @@ +blitiri.com.ar/go/spf v1.5.1 h1:CWUEasc44OrANJD8CzceRnRn1Jv0LttY68cYym2/pbE= +blitiri.com.ar/go/spf v1.5.1/go.mod h1:E71N92TfL4+Yyd5lpKuE9CAF2pd4JrUq1xQfkTxoNdk= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= +cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= +cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= +cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= +cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= +cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= +cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= +cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= +cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= +cloud.google.com/go/auth v0.14.0 h1:A5C4dKV/Spdvxcl0ggWwWEzzP7AZMJSEIgrkngwhGYM= +cloud.google.com/go/auth v0.14.0/go.mod h1:CYsoRL1PdiDuqeQpZE0bP2pnPrGqFcOkI0nldEQis+A= +cloud.google.com/go/auth/oauth2adapt v0.2.7 h1:/Lc7xODdqcEw8IrZ9SvwnlLX6j9FHQM74z6cBk9Rw6M= +cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc= +cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= +cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= +cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= +cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= +cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= +cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= +cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= +cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= +cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= +cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= +cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= +cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= +cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= +cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= +cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= +cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= +cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= +cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= +cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= +cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= +cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= +cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= +cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= +cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= +cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= +cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= +cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= +cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= +cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= +cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= +cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= +cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= +cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= +cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= +cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= +cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= +cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= +cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= +cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= +cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= +cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= +cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= +cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= +cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= +cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= +cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= +cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= +cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= +cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= +cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= +cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= +cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= +cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/G-Core/gcore-dns-sdk-go v0.2.9 h1:LMMZIRX8y3aJJuAviNSpFmLbovZUw+6Om+8VElp1F90= +github.com/G-Core/gcore-dns-sdk-go v0.2.9/go.mod h1:35t795gOfzfVanhzkFyUXEzaBuMXwETmJldPpP28MN4= +github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI= +github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI= +github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.44.40 h1:MR0qefjBJrZuXE0VoeKMQFtjS2tUeVpbQNfb7NzQNgI= +github.com/aws/aws-sdk-go v1.44.40/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go-v2 v1.33.0 h1:Evgm4DI9imD81V0WwD+TN4DCwjUMdc94TrduMLbgZJs= +github.com/aws/aws-sdk-go-v2 v1.33.0/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/config v1.29.1 h1:JZhGawAyZ/EuJeBtbQYnaoftczcb2drR2Iq36Wgz4sQ= +github.com/aws/aws-sdk-go-v2/config v1.29.1/go.mod h1:7bR2YD5euaxBhzt2y/oDkt3uNRb6tjFp98GlTFueRwk= +github.com/aws/aws-sdk-go-v2/credentials v1.17.54 h1:4UmqeOqJPvdvASZWrKlhzpRahAulBfyTJQUaYy4+hEI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.54/go.mod h1:RTdfo0P0hbbTxIhmQrOsC/PquBZGabEPnCaxxKRPSnI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 h1:5grmdTdMsovn9kPZPI23Hhvp0ZyNm5cRO+IZFIYiAfw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24/go.mod h1:zqi7TVKTswH3Ozq28PkmBmgzG1tona7mo9G2IJg4Cis= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 h1:igORFSiH3bfq4lxKFkTSYDhJEUCYo6C8VKiWJjYwQuQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28/go.mod h1:3So8EA/aAYm36L7XIvCVwLa0s5N0P7o2b1oqnx/2R4g= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 h1:1mOW9zAUMhTSrMDssEHS/ajx8JcAj/IcftzcmNlmVLI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28/go.mod h1:kGlXVIWDfvt2Ox5zEaNglmq0hXPHgQFNMix33Tw22jA= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 h1:TQmKDyETFGiXVhZfQ/I0cCFziqqX58pi4tKJGYGFSz0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9/go.mod h1:HVLPK2iHQBUx7HfZeOQSEu3v2ubZaAY2YPbAm5/WUyY= +github.com/aws/aws-sdk-go-v2/service/route53 v1.48.2 h1:Rxg1R0CHxVb9ggQLufOkr4an3yFEkTDN+N5+LFU4aEg= +github.com/aws/aws-sdk-go-v2/service/route53 v1.48.2/go.mod h1:TN4PcCL0lvqmYcv+AV8iZFC4Sd0FM06QDaoBXrFEftU= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 h1:kuIyu4fTT38Kj7YCC7ouNbVZSSpqkZ+LzIfhCr6Dg+I= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.11/go.mod h1:Ro744S4fKiCCuZECXgOi760TiYylUM8ZBf6OGiZzJtY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 h1:l+dgv/64iVlQ3WsBbnn+JSbkj01jIi+SM0wYsj3y/hY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10/go.mod h1:Fzsj6lZEb8AkTE5S68OhcbBqeWPsR8RnGuKPr8Todl8= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 h1:BRVDbewN6VZcwr+FBOszDKvYeXY1kJ+GGMCcpghlw0U= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.9/go.mod h1:f6vjfZER1M17Fokn0IzssOTMT2N8ZSq+7jnNF0tArvw= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/c0va23/go-proxyprotocol v0.9.1 h1:5BCkp0fDJOhzzH1lhjUgHhmZz9VvRMMif1U2D31hb34= +github.com/c0va23/go-proxyprotocol v0.9.1/go.mod h1:TNjUV+llvk8TvWJxlPYAeAYZgSzT/iicNr3nWBWX320= +github.com/caddyserver/certmagic v0.21.7 h1:66KJioPFJwttL43KYSWk7ErSmE6LfaJgCQuhm8Sg6fg= +github.com/caddyserver/certmagic v0.21.7/go.mod h1:LCPG3WLxcnjVKl/xpjzM0gqh0knrKKKiO5WVttX2eEI= +github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA= +github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/digitalocean/godo v1.41.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= +github.com/digitalocean/godo v1.134.0 h1:dT7aQR9jxNOQEZwzP+tAYcxlj5szFZScC33+PAYGQVM= +github.com/digitalocean/godo v1.134.0/go.mod h1:PU8JB6I1XYkQIdHFop8lLAY9ojp6M0XcU0TWaQSxbrc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emersion/go-imap-appendlimit v0.0.0-20190308131241-25671c986a6a/go.mod h1:ikgISoP7pRAolqsVP64yMteJa2FIpS6ju88eBT6K1yQ= +github.com/emersion/go-imap-compress v0.0.0-20201103190257-14809af1d1b9 h1:7dmV11mle4UAQ7lX+Hdzx6akKFg3hVm/UUmQ7t6VgTQ= +github.com/emersion/go-imap-compress v0.0.0-20201103190257-14809af1d1b9/go.mod h1:2Ro1PbmiqYiRe5Ct2sGR5hHaKSVHeRpVZwXx8vyYt98= +github.com/emersion/go-imap-move v0.0.0-20180601155324-5eb20cb834bf/go.mod h1:QuMaZcKFDVI0yCrnAbPLfbwllz1wtOrZH8/vZ5yzp4w= +github.com/emersion/go-imap-sortthread v1.2.0 h1:EMVEJXPWAhXMWECjR82Rn/tza6MddcvTwGAdTu1vJKU= +github.com/emersion/go-imap-sortthread v1.2.0/go.mod h1:UhenCBupR+vSYRnqJkpjSq84INUCsyAK1MLpogv14pE= +github.com/emersion/go-message v0.15.0/go.mod h1:wQUEfE+38+7EW8p8aZ96ptg6bAb1iwdgej19uXASlE4= +github.com/emersion/go-message v0.18.0/go.mod h1:Zi69ACvzaoV/MBnrxfVBPV3xWEuCmC2nEN39oJF4B8A= +github.com/emersion/go-message v0.18.1/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= +github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg= +github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= +github.com/emersion/go-milter v0.4.1 h1:gLs9QD0zEHF8omgEw8M+aGz6iwBNpWLAcwgSur0ra4M= +github.com/emersion/go-milter v0.4.1/go.mod h1:erCQVl0mH4SX9jEvwe+wyndit0rQtmvMLH86V6NGtkI= +github.com/emersion/go-msgauth v0.6.8 h1:kW/0E9E8Zx5CdKsERC/WnAvnXvX7q9wTHia1OA4944A= +github.com/emersion/go-msgauth v0.6.8/go.mod h1:YDwuyTCUHu9xxmAeVj0eW4INnwB6NNZoPdLerpSxRrc= +github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k= +github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3DfFi4EXdT+B+gTtzx8GC9xsc26Znk= +github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/foxcpp/go-dovecot-sasl v0.0.0-20200522223722-c4699d7a24bf h1:rmBPY5fryjp9zLQYsUmQqqgsYq7qeVfrjtr96Tf9vD8= +github.com/foxcpp/go-dovecot-sasl v0.0.0-20200522223722-c4699d7a24bf/go.mod h1:5yZUmwr851vgjyAfN7OEfnrmKOh/qLA5dbGelXYsu1E= +github.com/foxcpp/go-imap v1.0.0-beta.1.0.20220623182312-df940c324887 h1:qUoaaHyrRpQw85ru6VQcC6JowdhrWl7lSbI1zRX1FTM= +github.com/foxcpp/go-imap v1.0.0-beta.1.0.20220623182312-df940c324887/go.mod h1:Qlx1FSx2FTxjnjWpIlVNEuX+ylerZQNFE5NsmKFSejY= +github.com/foxcpp/go-imap-backend-tests v0.0.0-20220105184719-e80aa29a5e16 h1:qheFPDpteiUy7Ym18R68OYenpk85UyKYGkhYTmddSBg= +github.com/foxcpp/go-imap-backend-tests v0.0.0-20220105184719-e80aa29a5e16/go.mod h1:OPP1AgKxMPo3aHX5pcEZLQhhh5sllFcB8aUN9f6a6X8= +github.com/foxcpp/go-imap-i18nlevel v0.0.0-20200208001533-d6ec88553005 h1:pfoFtkTTQ473qStSN79jhCFBWqMQt/3DQ3NGuXvT+50= +github.com/foxcpp/go-imap-i18nlevel v0.0.0-20200208001533-d6ec88553005/go.mod h1:34FwxnjC2N+EFs2wMtsHevrZLWRKRuVU8wEcHWKq/nE= +github.com/foxcpp/go-imap-mess v0.0.0-20230108134257-b7ec3a649613 h1:fw9OWfPxP1CK4D+XAEEg0JzhvFGo04L+F5Xw55t9s3E= +github.com/foxcpp/go-imap-mess v0.0.0-20230108134257-b7ec3a649613/go.mod h1:P/O/qz4gaVkefzJ40BUtN/ZzBnaEg0YYe1no/SMp7Aw= +github.com/foxcpp/go-imap-namespace v0.0.0-20200802091432-08496dd8e0ed h1:1Jo7geyvunrPSjL6F6D9EcXoNApS5v3LQaro7aUNPnE= +github.com/foxcpp/go-imap-namespace v0.0.0-20200802091432-08496dd8e0ed/go.mod h1:Shows1vmkBWO40ChOClaUe6DUnZrsP1UPAuoWzIUdgQ= +github.com/foxcpp/go-imap-sql v0.5.1-0.20250124140007-8da5567429d5 h1:jMxhw9qmwqg70qfMDWq0ImRHAduQjkTZOC9vBs5t2ug= +github.com/foxcpp/go-imap-sql v0.5.1-0.20250124140007-8da5567429d5/go.mod h1:LMlfyNkVs7v2zE6OVeGe9qWPmKFdXDmLNddPLodPVIw= +github.com/foxcpp/go-mockdns v0.0.0-20191216195825-5eabd8dbfe1f/go.mod h1:tPg4cp4nseejPd+UKxtCVQ2hUxNTZ7qQZJa7CLriIeo= +github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI= +github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= +github.com/foxcpp/go-mtasts v0.0.0-20240130093538-1438da2e5932 h1:p04U/s8IZEc+PVWIDWGUgdqGq3xsixI7XRZ6Bp/xZbQ= +github.com/foxcpp/go-mtasts v0.0.0-20240130093538-1438da2e5932/go.mod h1:RtHIZCsScdjIzXpTTjmEljtUrIjQbPBTvw7F1tKQbKk= +github.com/foxcpp/go-smtp v1.21.4-0.20250124171104-c8519ae4fb23 h1:JSnsCrRrHNBlgfKVFBxFzp3fN/wS21t8fAHcZ9B1uWI= +github.com/foxcpp/go-smtp v1.21.4-0.20250124171104-c8519ae4fb23/go.mod h1:qm27SGYgoIPRot6ubfQ/GpiPy/g3PaZAVRxiO/sDUgQ= +github.com/foxcpp/libdns-gandi v1.0.4-0.20240127130558-4782f9d5ce3e h1:hKk+CGUtwnKDGKINPEojeo91kx0tnV6V4tlzHehJPfg= +github.com/foxcpp/libdns-gandi v1.0.4-0.20240127130558-4782f9d5ce3e/go.mod h1:G6dw58Xnji2xX+lb+uZxGbtmfxKllm1CGHE2bOPG3WA= +github.com/frankban/quicktest v1.5.0/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk= +github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-ldap/ldap/v3 v3.4.10 h1:ot/iwPOhfpNVgB1o+AVXljizWZ9JTp7YF5oeyONmcJU= +github.com/go-ldap/ldap/v3 v3.4.10/go.mod h1:JXh4Uxgi40P6E9rdsYqpUtbW46D9UTjJ9QSwGRznplY= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= +github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= +github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= +github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= +github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= +github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= +github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/johannesboyne/gofakes3 v0.0.0-20210704111953-6a9f95c2941c h1:lx/uPI+mUWlqEQ9e6CtNvaK/zD64s/mQ9+yMh16PgY0= +github.com/johannesboyne/gofakes3 v0.0.0-20210704111953-6a9f95c2941c/go.mod h1:LIAXxPvcUXwOcTIj9LSNSUpE9/eMHalTWxsP/kmWxQI= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libdns/acmedns v0.2.0 h1:zTXdHZwe3r2issdVRyqt5/4X2yHpiBVmFnTrwBA29ik= +github.com/libdns/acmedns v0.2.0/go.mod h1:XlKHilQQK/IGHYY//vCb903PdG4Wc/XnDQzcMp2hV3g= +github.com/libdns/alidns v1.0.3 h1:LFHuGnbseq5+HCeGa1aW8awyX/4M2psB9962fdD2+yQ= +github.com/libdns/alidns v1.0.3/go.mod h1:e18uAG6GanfRhcJj6/tps2rCMzQJaYVcGKT+ELjdjGE= +github.com/libdns/cloudflare v0.1.1 h1:FVPfWwP8zZCqj268LZjmkDleXlHPlFU9KC4OJ3yn054= +github.com/libdns/cloudflare v0.1.1/go.mod h1:9VK91idpOjg6v7/WbjkEW49bSCxj00ALesIFDhJ8PBU= +github.com/libdns/digitalocean v0.0.0-20230728223659-4f9064657aea h1:IGlMNZCUp8Ho7NYYorpP5ZJgg2mFXARs6eHs/pSqFkA= +github.com/libdns/digitalocean v0.0.0-20230728223659-4f9064657aea/go.mod h1:B2TChhOTxvBflpRTHlguXWtwa1Ha5WI6JkB6aCViM+0= +github.com/libdns/gcore v0.0.0-20250127070537-4a9d185c9d20 h1:bQwFw+C9sX/zYZlV53ey0KnNkxrfWYIFpvptuAVhJ1Y= +github.com/libdns/gcore v0.0.0-20250127070537-4a9d185c9d20/go.mod h1:JGoT1mbmqQwtYQqN5F/vGc9j4TTTMKw/hDm5vXADHUI= +github.com/libdns/googleclouddns v1.1.0 h1:murPR1LfTZZObLV2OLxUVmymWH25glkMFKpDjkk2m0E= +github.com/libdns/googleclouddns v1.1.0/go.mod h1:3tzd056dfqKlf71V8Oy19En4WjJ3ybyuWx6P9bQSCIw= +github.com/libdns/hetzner v0.0.1 h1:WsmcsOKnfpKmzwhfyqhGQEIlEeEaEUvb7ezoJgBKaqU= +github.com/libdns/hetzner v0.0.1/go.mod h1:Jj12aJipO9Ir7OGaXueJ5J1RnerFMD0auGa6k9kujG4= +github.com/libdns/leaseweb v0.4.0 h1:WG9R5AwewpYM4goymFwnG2SB0qwL8gMsSzwRHZHee/U= +github.com/libdns/leaseweb v0.4.0/go.mod h1:dvTvEn11JN6+ebhAQ60l+jiaBiEqyJFs3EIo0YBcQkU= +github.com/libdns/libdns v0.1.0/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= +github.com/libdns/libdns v0.2.0/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= +github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= +github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s= +github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= +github.com/libdns/metaname v0.3.0 h1:HJudLYthdv52TupOPczojip/nEQHW7xqk5+whGReva4= +github.com/libdns/metaname v0.3.0/go.mod h1:a3hqEgj59tjWaWlF4WxQGhvMVtjz1E4Ngs1GfVS+VhQ= +github.com/libdns/namecheap v0.0.0-20211109042440-fc7440785c8e h1:WCcKyxiiK/sJnST1ulVBKNg4J8luCYDdgUrp2ySMO2s= +github.com/libdns/namecheap v0.0.0-20211109042440-fc7440785c8e/go.mod h1:dED6sMLZxIcilF1GjrcpwgVoCglXGMn86irqQzRhqRY= +github.com/libdns/namedotcom v0.3.3 h1:R10C7+IqQGVeC4opHHMiFNBxdNBg1bi65ZwqLESl+jE= +github.com/libdns/namedotcom v0.3.3/go.mod h1:GbYzsAF2yRUpI0WgIK5fs5UX+kDVUPaYCFLpTnKQm0s= +github.com/libdns/rfc2136 v0.1.1 h1:GKh2r08xt4aYeGlXR9eFrJMfFKD5i9QHBOpT1FIww/U= +github.com/libdns/rfc2136 v0.1.1/go.mod h1:tgXWavE+5OiAfdKxBnuG8OBEwQFAu7uuiS3+laspAGs= +github.com/libdns/route53 v1.5.1 h1:dkdcc2CKY/EHBBzAKqE0Cko7MKR8uVJ3GvpzwKu/UKM= +github.com/libdns/route53 v1.5.1/go.mod h1:joT4hKmaTNKHEwb7GmZ65eoDz1whTu7KKYPS8ZqIh6Q= +github.com/libdns/vultr v1.0.0 h1:W8B4+k2bm9ro3bZLSZV9hMOQI+uO6Svu+GmD+Olz7ZI= +github.com/libdns/vultr v1.0.0/go.mod h1:8K1HJExcbeHS4YPkFHRZpqpXZzZ+DZAA0m0VikJgEqk= +github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM= +github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/martinlindhe/base36 v1.0.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mholt/acmez/v3 v3.0.1 h1:4PcjKjaySlgXK857aTfDuRbmnM5gb3Ruz3tvoSJAUp8= +github.com/mholt/acmez/v3 v3.0.1/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ= +github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= +github.com/miekg/dns v1.1.63 h1:8M5aAw6OMZfFXTT7K5V0Eu5YiiL8l7nUAkyN6C9YwaY= +github.com/miekg/dns v1.1.63/go.mod h1:6NGHfjhpmr5lt3XPLuyfDJi5AXbNIPM9PY6H6sF1Nfs= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.84 h1:D1HVmAF8JF8Bpi6IU4V9vIEj+8pc+xU88EWMs2yed0E= +github.com/minio/minio-go/v7 v7.0.84/go.mod h1:57YXpvc5l3rjPdhqNrDsvVlY0qPI6UTk1bflAe+9doY= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/netauth/netauth v0.6.2 h1:Gtx/Xxa6YUaGny+iVvWyp+FAmtLQ1IlbB2uWTZEpWxQ= +github.com/netauth/netauth v0.6.2/go.mod h1:4PEbISVqRCQaXaDAt289w3nK9UhoF8/ZOLy31Hbv7ds= +github.com/netauth/protocol v0.0.0-20210918062754-7fee492ffcbd h1:4yVpQ/+li28lQ/daYCWeDB08obRmjaoAw2qfFFaCQ40= +github.com/netauth/protocol v0.0.0-20210918062754-7fee492ffcbd/go.mod h1:wpK5wqysOJU1w2OxgG65du8M7UqBkxzsNaJdjwiRqAs= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= +github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= +github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 h1:J6qvD6rbmOil46orKqJaRPG+zTpoGlBTUdyv8ki63L0= +github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+VKSARF5y/tS9XFSP7vWDfS+GUC5vs/YT7M5XDTUEM= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= +github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= +github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= +github.com/vultr/govultr/v3 v3.14.1 h1:9BpyZgsWasuNoR39YVMcq44MSaF576Z4D+U3ro58eJQ= +github.com/vultr/govultr/v3 v3.14.1/go.mod h1:q34Wd76upKmf+vxFMgaNMH3A8BbsPBmSYZUGC8oZa5w= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI= +github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= +go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= +go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= +go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= +go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190310074541-c10a0554eabf/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= +golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= +golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= +google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.218.0 h1:x6JCjEWeZ9PFCRe9z0FBrNwj7pB7DOAqT35N+IPnAUA= +google.golang.org/api v0.218.0/go.mod h1:5VGHBAkxrA/8EFjLVEYmMUJ8/8+gWWQ3s4cFH0FxG2M= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= +google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= +google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= +google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221018160656-63c7b68cfc55/go.mod h1:45EK0dUbEZ2NHjCeAd2LXmyjAgGUGrpGROgjhC3ADck= +google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 h1:91mG8dNTpkC0uChJUQ9zCiRqx3GEEFOWaRZ0mI6Oj2I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= +google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= +google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0= +modernc.org/cc/v4 v4.24.4/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.23.13 h1:PFiaemQwE/jdwi8XEHyEV+qYWoIuikLP3T4rvDeJb00= +modernc.org/ccgo/v4 v4.23.13/go.mod h1:vdN4h2WR5aEoNondUx26K7G8X+nuBscYnAEWSRmN2/0= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.6.1 h1:+Qf6xdG8l7B27TQ8D8lw/iFMUj1RXRBOuMUWziJOsk8= +modernc.org/gc/v2 v2.6.1/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/libc v1.61.9 h1:PLSBXVkifXGELtJ5BOnBUyAHr7lsatNwFU/RRo4kfJM= +modernc.org/libc v1.61.9/go.mod h1:61xrnzk/aR8gr5bR7Uj/lLFLuXu2/zMpIjcry63Eumk= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.8.2 h1:cL9L4bcoAObu4NkxOlKWBWtNHIsnnACGF/TbqQ6sbcI= +modernc.org/memory v1.8.2/go.mod h1:ZbjSvMO5NQ1A2i3bWeDiVMxIorXwdClKE/0SZ+BMotU= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/README.md b/internal/README.md new file mode 100644 index 0000000..da5d57c --- /dev/null +++ b/internal/README.md @@ -0,0 +1,24 @@ +maddy source tree +------------------ + +Main maddy code base lives here. No packages are intended to be used in +third-party software hence API is not stable. + +Subdirectories are organized as follows: +``` +/ + auxiliary libraries +endpoint/ + modules - protocol listeners (e.g. SMTP server, etc) +target/ + modules - final delivery targets (including outbound delivery, such as + target.smtp, remote) +auth/ + modules - authentication providers +check/ + modules - message checkers (module.Check) +modify/ + modules - message modifiers (module.Modifier) +storage/ + modules - local messages storage implementations (module.Storage) +``` diff --git a/internal/auth/auth.go b/internal/auth/auth.go new file mode 100644 index 0000000..4df144c --- /dev/null +++ b/internal/auth/auth.go @@ -0,0 +1,53 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package auth + +import "strings" + +func CheckDomainAuth(username string, perDomain bool, allowedDomains []string) (loginName string, allowed bool) { + var accountName, domain string + if perDomain { + parts := strings.Split(username, "@") + if len(parts) != 2 { + return "", false + } + domain = parts[1] + accountName = username + } else { + parts := strings.Split(username, "@") + accountName = parts[0] + if len(parts) == 2 { + domain = parts[1] + } + } + + allowed = domain == "" + if allowedDomains != nil && domain != "" { + for _, allowedDomain := range allowedDomains { + if strings.EqualFold(domain, allowedDomain) { + allowed = true + } + } + if !allowed { + return "", false + } + } + + return accountName, allowed +} diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go new file mode 100644 index 0000000..65ffee4 --- /dev/null +++ b/internal/auth/auth_test.go @@ -0,0 +1,92 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package auth + +import ( + "fmt" + "testing" +) + +func TestCheckDomainAuth(t *testing.T) { + cases := []struct { + rawUsername string + + perDomain bool + allowedDomains []string + + loginName string + }{ + { + rawUsername: "username", + loginName: "username", + }, + { + rawUsername: "username", + allowedDomains: []string{"example.org"}, + loginName: "username", + }, + { + rawUsername: "username@example.org", + allowedDomains: []string{"example.org"}, + loginName: "username", + }, + { + rawUsername: "username@example.com", + allowedDomains: []string{"example.org"}, + }, + { + rawUsername: "username", + allowedDomains: []string{"example.org"}, + perDomain: true, + }, + { + rawUsername: "username@example.com", + allowedDomains: []string{"example.org"}, + perDomain: true, + }, + { + rawUsername: "username@EXAMPLE.Org", + allowedDomains: []string{"exaMPle.org"}, + perDomain: true, + loginName: "username@EXAMPLE.Org", + }, + { + rawUsername: "username@example.org", + allowedDomains: []string{"example.org"}, + perDomain: true, + loginName: "username@example.org", + }, + } + + for _, case_ := range cases { + t.Run(fmt.Sprintf("%+v", case_), func(t *testing.T) { + loginName, allowed := CheckDomainAuth(case_.rawUsername, case_.perDomain, case_.allowedDomains) + if case_.loginName != "" && !allowed { + t.Fatalf("Unexpected authentication fail") + } + if case_.loginName == "" && allowed { + t.Fatalf("Expected authentication fail, got %s as login name", loginName) + } + + if loginName != case_.loginName { + t.Errorf("Incorrect login name, got %s, wanted %s", loginName, case_.loginName) + } + }) + } +} diff --git a/internal/auth/dovecot_sasl/dovecot_sasl.go b/internal/auth/dovecot_sasl/dovecot_sasl.go new file mode 100644 index 0000000..c7dd6cc --- /dev/null +++ b/internal/auth/dovecot_sasl/dovecot_sasl.go @@ -0,0 +1,160 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dovecotsasl + +import ( + "fmt" + "net" + + "github.com/emersion/go-sasl" + dovecotsasl "github.com/foxcpp/go-dovecot-sasl" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth" +) + +type Auth struct { + instName string + serverEndpoint string + log log.Logger + + network string + addr string + + mechanisms map[string]dovecotsasl.Mechanism +} + +const modName = "dovecot_sasl" + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + a := &Auth{ + instName: instName, + log: log.Logger{Name: modName, Debug: log.DefaultLogger.Debug}, + } + + switch len(inlineArgs) { + case 0: + case 1: + a.serverEndpoint = inlineArgs[0] + default: + return nil, fmt.Errorf("%s: one or none arguments needed", modName) + } + + return a, nil +} + +func (a *Auth) Name() string { + return modName +} + +func (a *Auth) InstanceName() string { + return a.instName +} + +func (a *Auth) getConn() (*dovecotsasl.Client, error) { + // TODO: Connection pooling + conn, err := net.Dial(a.network, a.addr) + if err != nil { + return nil, fmt.Errorf("%s: unable to contact server: %v", modName, err) + } + + cl, err := dovecotsasl.NewClient(conn) + if err != nil { + return nil, fmt.Errorf("%s: unable to contact server: %v", modName, err) + } + + return cl, nil +} + +func (a *Auth) returnConn(cl *dovecotsasl.Client) { + cl.Close() +} + +func (a *Auth) Init(cfg *config.Map) error { + cfg.String("endpoint", false, false, a.serverEndpoint, &a.serverEndpoint) + if _, err := cfg.Process(); err != nil { + return err + } + if a.serverEndpoint == "" { + return fmt.Errorf("%s: missing server endpoint", modName) + } + + endp, err := config.ParseEndpoint(a.serverEndpoint) + if err != nil { + return fmt.Errorf("%s: invalid server endpoint: %v", modName, err) + } + + // Dial once to check usability and also to get list of mechanisms. + conn, err := net.Dial(endp.Scheme, endp.Address()) + if err != nil { + return fmt.Errorf("%s: unable to contact server: %v", modName, err) + } + + cl, err := dovecotsasl.NewClient(conn) + if err != nil { + return fmt.Errorf("%s: unable to contact server: %v", modName, err) + } + + defer cl.Close() + a.mechanisms = make(map[string]dovecotsasl.Mechanism, len(cl.ConnInfo().Mechs)) + for name, mech := range cl.ConnInfo().Mechs { + if mech.Private { + continue + } + a.mechanisms[name] = mech + } + + a.network = endp.Scheme + a.addr = endp.Address() + + return nil +} + +func (a *Auth) AuthPlain(username, password string) error { + if _, ok := a.mechanisms[sasl.Plain]; ok { + cl, err := a.getConn() + if err != nil { + return exterrors.WithTemporary(err, true) + } + defer a.returnConn(cl) + + // Pretend it is SMTPS even though we really don't know. + // We also have no connection information to pass to the server... + return cl.Do("SMTP", sasl.NewPlainClient("", username, password), + dovecotsasl.Secured, dovecotsasl.NoPenalty) + } + if _, ok := a.mechanisms[sasl.Login]; ok { + cl, err := a.getConn() + if err != nil { + return err + } + defer a.returnConn(cl) + + return cl.Do("SMTP", sasl.NewLoginClient(username, password), + dovecotsasl.Secured, dovecotsasl.NoPenalty) + } + + return auth.ErrUnsupportedMech +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/auth/external/externalauth.go b/internal/auth/external/externalauth.go new file mode 100644 index 0000000..59d71fb --- /dev/null +++ b/internal/auth/external/externalauth.go @@ -0,0 +1,103 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package external + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth" +) + +type ExternalAuth struct { + modName string + instName string + helperPath string + + perDomain bool + domains []string + + Log log.Logger +} + +func NewExternalAuth(modName, instName string, _, inlineArgs []string) (module.Module, error) { + ea := &ExternalAuth{ + modName: modName, + instName: instName, + Log: log.Logger{Name: modName}, + } + + if len(inlineArgs) != 0 { + return nil, errors.New("external: inline arguments are not used") + } + + return ea, nil +} + +func (ea *ExternalAuth) Name() string { + return ea.modName +} + +func (ea *ExternalAuth) InstanceName() string { + return ea.instName +} + +func (ea *ExternalAuth) Init(cfg *config.Map) error { + cfg.Bool("debug", false, false, &ea.Log.Debug) + cfg.Bool("perdomain", false, false, &ea.perDomain) + cfg.StringList("domains", false, false, nil, &ea.domains) + cfg.String("helper", false, false, "", &ea.helperPath) + if _, err := cfg.Process(); err != nil { + return err + } + if ea.perDomain && ea.domains == nil { + return errors.New("auth_domains must be set if auth_perdomain is used") + } + + if ea.helperPath != "" { + ea.Log.Debugln("using helper:", ea.helperPath) + } else { + ea.helperPath = filepath.Join(config.LibexecDirectory, "maddy-auth-helper") + } + if _, err := os.Stat(ea.helperPath); err != nil { + return fmt.Errorf("%s doesn't exist", ea.helperPath) + } + + ea.Log.Debugln("using helper:", ea.helperPath) + + return nil +} + +func (ea *ExternalAuth) AuthPlain(username, password string) error { + accountName, ok := auth.CheckDomainAuth(username, ea.perDomain, ea.domains) + if !ok { + return module.ErrUnknownCredentials + } + + return AuthUsingHelper(ea.helperPath, accountName, password) +} + +func init() { + module.Register("auth.external", NewExternalAuth) +} diff --git a/internal/auth/external/helperauth.go b/internal/auth/external/helperauth.go new file mode 100644 index 0000000..901d57d --- /dev/null +++ b/internal/auth/external/helperauth.go @@ -0,0 +1,55 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package external + +import ( + "fmt" + "io" + "os/exec" + + "github.com/foxcpp/maddy/framework/module" +) + +func AuthUsingHelper(binaryPath, accountName, password string) error { + cmd := exec.Command(binaryPath) + stdin, err := cmd.StdinPipe() + if err != nil { + return fmt.Errorf("helperauth: stdin init: %w", err) + } + if err := cmd.Start(); err != nil { + return fmt.Errorf("helperauth: process start: %w", err) + } + if _, err := io.WriteString(stdin, accountName+"\n"); err != nil { + return fmt.Errorf("helperauth: stdin write: %w", err) + } + if _, err := io.WriteString(stdin, password+"\n"); err != nil { + return fmt.Errorf("helperauth: stdin write: %w", err) + } + if err := cmd.Wait(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + // Exit code 1 is for authentication failure. + if exitErr.ExitCode() != 1 { + return fmt.Errorf("helperauth: %w: %v", err, string(exitErr.Stderr)) + } + return module.ErrUnknownCredentials + } + return fmt.Errorf("helperauth: process wait: %w", err) + } + return nil +} diff --git a/internal/auth/ldap/ldap.go b/internal/auth/ldap/ldap.go new file mode 100644 index 0000000..04cfe9f --- /dev/null +++ b/internal/auth/ldap/ldap.go @@ -0,0 +1,289 @@ +package ldap + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/url" + "strings" + "sync" + "time" + + "github.com/foxcpp/maddy/framework/config" + tls2 "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/go-ldap/ldap/v3" +) + +const modName = "auth.ldap" + +type Auth struct { + instName string + + urls []string + readBind func(*ldap.Conn) error + startls bool + tlsCfg tls.Config + dialer *net.Dialer + requestTimeout time.Duration + + dnTemplate string + // or + baseDN string + filterTemplate string + + conn *ldap.Conn + connLock sync.Mutex + + log log.Logger +} + +func New(modName, instName string, _, inlineArgs []string) (module.Module, error) { + return &Auth{ + instName: instName, + log: log.Logger{Name: modName}, + urls: inlineArgs, + }, nil +} + +func (a *Auth) Init(cfg *config.Map) error { + a.dialer = &net.Dialer{} + + cfg.Bool("debug", true, false, &a.log.Debug) + cfg.Custom("tls_client", true, false, func() (interface{}, error) { + return tls.Config{}, nil + }, tls2.TLSClientBlock, &a.tlsCfg) + cfg.Callback("urls", func(m *config.Map, node config.Node) error { + a.urls = append(a.urls, node.Args...) + return nil + }) + cfg.Custom("bind", false, false, func() (interface{}, error) { + return func(*ldap.Conn) error { + return nil + }, nil + }, readBindDirective, &a.readBind) + cfg.Bool("starttls", false, false, &a.startls) + cfg.Duration("connect_timeout", false, false, time.Minute, &a.dialer.Timeout) + cfg.Duration("request_timeout", false, false, time.Minute, &a.requestTimeout) + cfg.String("dn_template", false, false, "", &a.dnTemplate) + cfg.String("base_dn", false, false, "", &a.baseDN) + cfg.String("filter", false, false, "", &a.filterTemplate) + if _, err := cfg.Process(); err != nil { + return err + } + + if a.dnTemplate == "" { + if a.baseDN == "" { + return fmt.Errorf("auth.ldap: base_dn not set") + } + if a.filterTemplate == "" { + return fmt.Errorf("auth.ldap: filter not set") + } + } else { + if a.baseDN != "" || a.filterTemplate != "" { + return fmt.Errorf("auth.ldap: search directives set when dn_template is used") + } + } + + if module.NoRun { + return nil + } + + var err error + a.conn, err = a.newConn() + if err != nil { + return fmt.Errorf("auth.ldap: %w", err) + } + return nil +} + +func readBindDirective(c *config.Map, n config.Node) (interface{}, error) { + if len(n.Args) == 0 { + return nil, fmt.Errorf("auth.ldap: auth expects at least one argument") + } + switch n.Args[0] { + case "off": + return func(*ldap.Conn) error { return nil }, nil + case "unauth": + if len(n.Args) == 2 { + return func(c *ldap.Conn) error { + return c.UnauthenticatedBind(n.Args[1]) + }, nil + } + return func(c *ldap.Conn) error { + return c.UnauthenticatedBind("") + }, nil + case "plain": + if len(n.Args) != 3 { + return nil, fmt.Errorf("auth.ldap: username and password expected for plaintext bind") + } + return func(c *ldap.Conn) error { + return c.Bind(n.Args[1], n.Args[2]) + }, nil + case "external": + return (*ldap.Conn).ExternalBind, nil + } + return nil, fmt.Errorf("auth.ldap: unknown bind authentication: %v", n.Args[0]) +} + +func (a *Auth) Name() string { + return modName +} + +func (a *Auth) InstanceName() string { + return a.instName +} + +func (a *Auth) newConn() (*ldap.Conn, error) { + var ( + conn *ldap.Conn + tlsCfg *tls.Config + ) + for _, u := range a.urls { + parsedURL, err := url.Parse(u) + if err != nil { + return nil, fmt.Errorf("auth.ldap: invalid server URL: %w", err) + } + hostname := parsedURL.Host + a.tlsCfg.ServerName = strings.Split(hostname, ":")[0] + tlsCfg = a.tlsCfg.Clone() + + conn, err = ldap.DialURL(u, ldap.DialWithDialer(a.dialer), ldap.DialWithTLSConfig(tlsCfg)) + if err != nil { + a.log.Error("cannot contact directory server", err, "url", u) + continue + } + break + } + if conn == nil { + return nil, fmt.Errorf("auth.ldap: all directory servers are unreachable") + } + + if a.requestTimeout != 0 { + conn.SetTimeout(a.requestTimeout) + } + + if a.startls { + if err := conn.StartTLS(tlsCfg); err != nil { + return nil, fmt.Errorf("auth.ldap: %w", err) + } + } + + if err := a.readBind(conn); err != nil { + return nil, fmt.Errorf("auth.ldap: %w", err) + } + + return conn, nil +} + +func (a *Auth) getConn() (*ldap.Conn, error) { + a.connLock.Lock() + if a.conn == nil { + conn, err := a.newConn() + if err != nil { + a.connLock.Unlock() + return nil, err + } + a.conn = conn + } + if a.conn.IsClosing() { + a.conn.Close() + conn, err := a.newConn() + if err != nil { + a.connLock.Unlock() + return nil, err + } + a.conn = conn + } + return a.conn, nil +} + +func (a *Auth) returnConn(conn *ldap.Conn) { + defer a.connLock.Unlock() + if err := a.readBind(conn); err != nil { + a.log.Error("failed to rebind for reading", err) + conn.Close() + a.conn = nil + } + if a.conn != conn { + a.conn.Close() + } + a.conn = conn +} + +func (a *Auth) Lookup(_ context.Context, username string) (string, bool, error) { + conn, err := a.getConn() + if err != nil { + return "", false, err + } + defer a.returnConn(conn) + + var userDN string + if a.dnTemplate != "" { + return "", false, fmt.Errorf("auth.ldap: lookups require search config but dn_template is used") + } else { + req := ldap.NewSearchRequest( + a.baseDN, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, + 2, 0, false, + strings.ReplaceAll(a.filterTemplate, "{username}", username), + []string{"dn"}, nil) + res, err := conn.Search(req) + if err != nil { + return "", false, fmt.Errorf("auth.ldap: search: %w", err) + } + if len(res.Entries) > 1 { + return "", false, fmt.Errorf("auth.ldap: too manu entries returned (%d)", len(res.Entries)) + } + if len(res.Entries) == 0 { + return "", false, nil + } + userDN = res.Entries[0].DN + } + + return userDN, true, nil +} + +func (a *Auth) AuthPlain(username, password string) error { + conn, err := a.getConn() + if err != nil { + return err + } + defer a.returnConn(conn) + + var userDN string + if a.dnTemplate != "" { + userDN = strings.ReplaceAll(a.dnTemplate, "{username}", username) + } else { + req := ldap.NewSearchRequest( + a.baseDN, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, + 2, 0, false, + strings.ReplaceAll(a.filterTemplate, "{username}", username), + []string{"dn"}, nil) + res, err := conn.Search(req) + if err != nil { + return fmt.Errorf("auth.ldap: search: %w", err) + } + if len(res.Entries) > 1 { + return fmt.Errorf("auth.ldap: too manu entries returned (%d)", len(res.Entries)) + } + if len(res.Entries) == 0 { + return module.ErrUnknownCredentials + } + userDN = res.Entries[0].DN + } + + if err := conn.Bind(userDN, password); err != nil { + return module.ErrUnknownCredentials + } + + return nil +} + +func init() { + var _ module.PlainAuth = &Auth{} + var _ module.Table = &Auth{} + module.Register(modName, New) + module.Register("table.ldap", New) +} diff --git a/internal/auth/netauth/netauth.go b/internal/auth/netauth/netauth.go new file mode 100644 index 0000000..3348eef --- /dev/null +++ b/internal/auth/netauth/netauth.go @@ -0,0 +1,117 @@ +package netauth + +import ( + "context" + "fmt" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/hashicorp/go-hclog" + "github.com/netauth/netauth/pkg/netauth" +) + +const modName = "auth.netauth" + +func init() { + var _ module.PlainAuth = &Auth{} + var _ module.Table = &Auth{} + module.Register(modName, New) + module.Register("table.netauth", New) +} + +// Auth binds all methods related to the NetAuth client library. +type Auth struct { + instName string + mustGroup string + + nacl *netauth.Client + + log log.Logger +} + +// New creates a new instance of the NetAuth module. +func New(modName, instName string, _, inlineArgs []string) (module.Module, error) { + return &Auth{ + instName: instName, + log: log.Logger{Name: modName}, + }, nil +} + +// Init performs deferred initialization actions. +func (a *Auth) Init(cfg *config.Map) error { + l := hclog.New(&hclog.LoggerOptions{Output: a.log}) + n, err := netauth.NewWithLog(l) + if err != nil { + return err + } + a.nacl = n + a.nacl.SetServiceName("maddy") + cfg.String("require_group", false, false, "", &a.mustGroup) + cfg.Bool("debug", true, false, &a.log.Debug) + if _, err := cfg.Process(); err != nil { + return err + } + + a.log.Debugln("Debug logging enabled") + a.log.Debugf("mustGroups status: %s", a.mustGroup) + return nil +} + +// Name returns "auth.netauth" as the fixed module name. +func (a *Auth) Name() string { + return modName +} + +// InstanceName returns the configured name for this instance of the +// plugin. Given the way that NetAuth works it doesn't really make +// sense to have more than one instance, but this is part of the API. +func (a *Auth) InstanceName() string { + return a.instName +} + +// Lookup requests the entity from the remote NetAuth server, +// potentially returning that the user does not exist at all. +func (a *Auth) Lookup(ctx context.Context, username string) (string, bool, error) { + e, err := a.nacl.EntityInfo(ctx, username) + if err != nil { + return "", false, fmt.Errorf("%s: search: %w", modName, err) + } + + if a.mustGroup != "" { + if err := a.checkMustGroup(username); err != nil { + return "", false, err + } + } + return e.GetID(), true, nil +} + +// AuthPlain attempts straightforward authentication of the entity on +// the remote NetAuth server. +func (a *Auth) AuthPlain(username, password string) error { + a.log.Debugf("attempting to auth user: %s", username) + if err := a.nacl.AuthEntity(context.Background(), username, password); err != nil { + return module.ErrUnknownCredentials + } + a.log.Debugln("netauth returns successful auth") + if a.mustGroup != "" { + if err := a.checkMustGroup(username); err != nil { + return err + } + } + return nil +} + +func (a *Auth) checkMustGroup(username string) error { + a.log.Debugf("Performing require_group check: must=%s", a.mustGroup) + groups, err := a.nacl.EntityGroups(context.Background(), username) + if err != nil { + return fmt.Errorf("%s: groups: %w", modName, err) + } + for _, g := range groups { + if g.GetName() == a.mustGroup { + return nil + } + } + return fmt.Errorf("%s: missing required group (%s not in %s)", modName, username, a.mustGroup) +} diff --git a/internal/auth/pam/module.go b/internal/auth/pam/module.go new file mode 100644 index 0000000..c93269d --- /dev/null +++ b/internal/auth/pam/module.go @@ -0,0 +1,94 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package pam + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth/external" +) + +type Auth struct { + instName string + useHelper bool + helperPath string + + Log log.Logger +} + +func New(modName, instName string, _, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, errors.New("pam: inline arguments are not used") + } + return &Auth{ + instName: instName, + Log: log.Logger{Name: modName}, + }, nil +} + +func (a *Auth) Name() string { + return "pam" +} + +func (a *Auth) InstanceName() string { + return a.instName +} + +func (a *Auth) Init(cfg *config.Map) error { + cfg.Bool("debug", true, false, &a.Log.Debug) + cfg.Bool("use_helper", false, false, &a.useHelper) + if _, err := cfg.Process(); err != nil { + return err + } + if !canCallDirectly && !a.useHelper { + return errors.New("pam: this build lacks support for direct libpam invocation, use helper binary") + } + + if a.useHelper { + a.helperPath = filepath.Join(config.LibexecDirectory, "maddy-pam-helper") + if _, err := os.Stat(a.helperPath); err != nil { + return fmt.Errorf("pam: no helper binary (maddy-pam-helper) found in %s", config.LibexecDirectory) + } + } + + return nil +} + +func (a *Auth) AuthPlain(username, password string) error { + if a.useHelper { + if err := external.AuthUsingHelper(a.helperPath, username, password); err != nil { + return err + } + } + err := runPAMAuth(username, password) + if err != nil { + return err + } + return nil +} + +func init() { + module.Register("auth.pam", New) +} diff --git a/internal/auth/pam/pam.c b/internal/auth/pam/pam.c new file mode 100644 index 0000000..38e9942 --- /dev/null +++ b/internal/auth/pam/pam.c @@ -0,0 +1,102 @@ +//+build libpam + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2022 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include "pam.h" + +static int conv_func(int num_msg, const struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) { + struct pam_response *reply = malloc(sizeof(struct pam_response)); + if (reply == NULL) { + return PAM_CONV_ERR; + } + + char* password_cpy = malloc(strlen((char*)appdata_ptr)+1); + if (password_cpy == NULL) { + return PAM_CONV_ERR; + } + memcpy(password_cpy, (char*)appdata_ptr, strlen((char*)appdata_ptr)+1); + + reply->resp = password_cpy; + reply->resp_retcode = 0; + + // PAM frees pam_response for us. + *resp = reply; + + return PAM_SUCCESS; +} + +struct error_obj run_pam_auth(const char *username, char *password) { + const struct pam_conv local_conv = { conv_func, password }; + pam_handle_t *local_auth = NULL; + int status = pam_start("maddy", username, &local_conv, &local_auth); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + ret_val.status = 2; + ret_val.func_name = "pam_start"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + status = pam_authenticate(local_auth, PAM_SILENT|PAM_DISALLOW_NULL_AUTHTOK); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + if (status == PAM_AUTH_ERR || status == PAM_USER_UNKNOWN) { + ret_val.status = 1; + } else { + ret_val.status = 2; + } + ret_val.func_name = "pam_authenticate"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + status = pam_acct_mgmt(local_auth, PAM_SILENT|PAM_DISALLOW_NULL_AUTHTOK); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + if (status == PAM_AUTH_ERR || status == PAM_USER_UNKNOWN || status == PAM_NEW_AUTHTOK_REQD) { + ret_val.status = 1; + } else { + ret_val.status = 2; + } + ret_val.func_name = "pam_acct_mgmt"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + status = pam_end(local_auth, status); + if (status != PAM_SUCCESS) { + struct error_obj ret_val; + ret_val.status = 2; + ret_val.func_name = "pam_end"; + ret_val.error_msg = pam_strerror(local_auth, status); + return ret_val; + } + + struct error_obj ret_val; + ret_val.status = 0; + ret_val.func_name = NULL; + ret_val.error_msg = NULL; + return ret_val; +} + diff --git a/internal/auth/pam/pam.go b/internal/auth/pam/pam.go new file mode 100644 index 0000000..2b5b0ef --- /dev/null +++ b/internal/auth/pam/pam.go @@ -0,0 +1,56 @@ +//go:build cgo && libpam +// +build cgo,libpam + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package pam + +/* +#cgo LDFLAGS: -lpam +#cgo CFLAGS: -DCGO -Wall -Wextra -Werror -Wno-unused-parameter -Wno-error=unused-parameter -Wpedantic -std=c99 + +#include +#include "pam.h" +*/ +import "C" + +import ( + "errors" + "fmt" + "unsafe" +) + +const canCallDirectly = true + +var ErrInvalidCredentials = errors.New("pam: invalid credentials or unknown user") + +func runPAMAuth(username, password string) error { + usernameC := C.CString(username) + passwordC := C.CString(password) + defer C.free(unsafe.Pointer(usernameC)) + defer C.free(unsafe.Pointer(passwordC)) + errObj := C.run_pam_auth(usernameC, passwordC) + if errObj.status == 1 { + return ErrInvalidCredentials + } + if errObj.status == 2 { + return fmt.Errorf("%s: %s", C.GoString(errObj.func_name), C.GoString(errObj.error_msg)) + } + return nil +} diff --git a/internal/auth/pam/pam.h b/internal/auth/pam/pam.h new file mode 100644 index 0000000..e9831ec --- /dev/null +++ b/internal/auth/pam/pam.h @@ -0,0 +1,27 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +#pragma once + +struct error_obj { + int status; + const char* func_name; + const char* error_msg; +}; + +struct error_obj run_pam_auth(const char *username, char *password); diff --git a/internal/auth/pam/pam_stub.go b/internal/auth/pam/pam_stub.go new file mode 100644 index 0000000..fb75421 --- /dev/null +++ b/internal/auth/pam/pam_stub.go @@ -0,0 +1,34 @@ +//go:build !cgo || !libpam +// +build !cgo !libpam + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package pam + +import ( + "errors" +) + +const canCallDirectly = false + +var ErrInvalidCredentials = errors.New("pam: invalid credentials or unknown user") + +func runPAMAuth(username, password string) error { + return errors.New("pam: Can't call libpam directly") +} diff --git a/internal/auth/pass_table/hash.go b/internal/auth/pass_table/hash.go new file mode 100644 index 0000000..94d151c --- /dev/null +++ b/internal/auth/pass_table/hash.go @@ -0,0 +1,180 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package pass_table + +import ( + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "fmt" + "io" + "strconv" + "strings" + + "golang.org/x/crypto/argon2" + "golang.org/x/crypto/bcrypt" +) + +const ( + HashSHA256 = "sha256" + HashBcrypt = "bcrypt" + HashArgon2 = "argon2" + + DefaultHash = HashBcrypt + + Argon2Salt = 16 + Argon2Size = 64 +) + +type ( + // HashOpts is the structure that holds additional parameters for used hash + // functions. They are used for new passwords. + // + // These parameters should be stored together with the hashed password + // so it can be verified independently of the used HashOpts. + HashOpts struct { + // Bcrypt cost value to use. Should be at least 10. + BcryptCost int + + Argon2Time uint32 + Argon2Memory uint32 + Argon2Threads uint8 + } + + FuncHashCompute func(opts HashOpts, pass string) (string, error) + FuncHashVerify func(pass, hashSalt string) error +) + +var ( + HashCompute = map[string]FuncHashCompute{ + HashBcrypt: computeBcrypt, + HashArgon2: computeArgon2, + } + HashVerify = map[string]FuncHashVerify{ + HashBcrypt: verifyBcrypt, + HashArgon2: verifyArgon2, + } + + Hashes = []string{HashSHA256, HashBcrypt, HashArgon2} +) + +func computeArgon2(opts HashOpts, pass string) (string, error) { + salt := make([]byte, Argon2Salt) + if _, err := io.ReadFull(rand.Reader, salt); err != nil { + return "", fmt.Errorf("pass_table: failed to generate salt: %w", err) + } + + hash := argon2.IDKey([]byte(pass), salt, opts.Argon2Time, opts.Argon2Memory, opts.Argon2Threads, Argon2Size) + var out strings.Builder + out.WriteString(strconv.FormatUint(uint64(opts.Argon2Time), 10)) + out.WriteRune(':') + out.WriteString(strconv.FormatUint(uint64(opts.Argon2Memory), 10)) + out.WriteRune(':') + out.WriteString(strconv.FormatUint(uint64(opts.Argon2Threads), 10)) + out.WriteRune(':') + out.WriteString(base64.StdEncoding.EncodeToString(salt)) + out.WriteRune(':') + out.WriteString(base64.StdEncoding.EncodeToString(hash)) + return out.String(), nil +} + +func verifyArgon2(pass, hashSalt string) error { + parts := strings.SplitN(hashSalt, ":", 5) + + time, err := strconv.ParseUint(parts[0], 10, 32) + if err != nil { + return fmt.Errorf("pass_table: malformed hash string: %w", err) + } + memory, err := strconv.ParseUint(parts[1], 10, 32) + if err != nil { + return fmt.Errorf("pass_table: malformed hash string: %w", err) + } + threads, err := strconv.ParseUint(parts[2], 10, 8) + if err != nil { + return fmt.Errorf("pass_table: malformed hash string: %w", err) + } + salt, err := base64.StdEncoding.DecodeString(parts[3]) + if err != nil { + return fmt.Errorf("pass_table: malformed hash string: %w", err) + } + hash, err := base64.StdEncoding.DecodeString(parts[4]) + if err != nil { + return fmt.Errorf("pass_table: malformed hash string: %w", err) + } + + passHash := argon2.IDKey([]byte(pass), salt, uint32(time), uint32(memory), uint8(threads), Argon2Size) + if subtle.ConstantTimeCompare(passHash, hash) != 1 { + return fmt.Errorf("pass_table: hash mismatch") + } + return nil +} + +func computeSHA256(_ HashOpts, pass string) (string, error) { + salt := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, salt); err != nil { + return "", fmt.Errorf("pass_table: failed to generate salt: %w", err) + } + + hashInput := salt + hashInput = append(hashInput, []byte(pass)...) + sum := sha256.Sum256(hashInput) + return base64.StdEncoding.EncodeToString(salt) + ":" + base64.StdEncoding.EncodeToString(sum[:]), nil +} + +func verifySHA256(pass, hashSalt string) error { + parts := strings.Split(hashSalt, ":") + if len(parts) != 2 { + return fmt.Errorf("pass_table: malformed hash string, no salt") + } + salt, err := base64.StdEncoding.DecodeString(parts[0]) + if err != nil { + return fmt.Errorf("pass_table: malformed hash string, cannot decode pass: %w", err) + } + hash, err := base64.StdEncoding.DecodeString(parts[1]) + if err != nil { + return fmt.Errorf("pass_table: malformed hash string, cannot decode pass: %w", err) + } + + hashInput := salt + hashInput = append(hashInput, []byte(pass)...) + sum := sha256.Sum256(hashInput) + + if subtle.ConstantTimeCompare(sum[:], hash) != 1 { + return fmt.Errorf("pass_table: hash mismatch") + } + return nil +} + +func computeBcrypt(opts HashOpts, pass string) (string, error) { + hash, err := bcrypt.GenerateFromPassword([]byte(pass), opts.BcryptCost) + if err != nil { + return "", err + } + return string(hash), nil +} + +func verifyBcrypt(pass, hashSalt string) error { + return bcrypt.CompareHashAndPassword([]byte(hashSalt), []byte(pass)) +} + +func addSHA256() { + HashCompute[HashSHA256] = computeSHA256 + HashVerify[HashSHA256] = verifySHA256 +} diff --git a/internal/auth/pass_table/table.go b/internal/auth/pass_table/table.go new file mode 100644 index 0000000..4d0e313 --- /dev/null +++ b/internal/auth/pass_table/table.go @@ -0,0 +1,198 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package pass_table + +import ( + "context" + "fmt" + "strings" + + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/module" + "golang.org/x/crypto/bcrypt" + "golang.org/x/text/secure/precis" +) + +type Auth struct { + modName string + instName string + inlineArgs []string + + table module.Table +} + +func New(modName, instName string, _, inlineArgs []string) (module.Module, error) { + return &Auth{ + modName: modName, + instName: instName, + inlineArgs: inlineArgs, + }, nil +} + +func (a *Auth) Init(cfg *config.Map) error { + if len(a.inlineArgs) != 0 { + return modconfig.ModuleFromNode("table", a.inlineArgs, cfg.Block, cfg.Globals, &a.table) + } + + cfg.Custom("table", false, true, nil, modconfig.TableDirective, &a.table) + _, err := cfg.Process() + return err +} + +func (a *Auth) Name() string { + return a.modName +} + +func (a *Auth) InstanceName() string { + return a.instName +} + +func (a *Auth) Lookup(ctx context.Context, username string) (string, bool, error) { + key, err := precis.UsernameCaseMapped.CompareKey(username) + if err != nil { + return "", false, err + } + + return a.table.Lookup(ctx, key) +} + +func (a *Auth) AuthPlain(username, password string) error { + key, err := precis.UsernameCaseMapped.CompareKey(username) + if err != nil { + return err + } + + hash, ok, err := a.table.Lookup(context.TODO(), key) + if !ok { + return module.ErrUnknownCredentials + } + if err != nil { + return err + } + + parts := strings.SplitN(hash, ":", 2) + if len(parts) != 2 { + return fmt.Errorf("%s: auth plain %s: no hash tag", a.modName, key) + } + hashVerify := HashVerify[parts[0]] + if hashVerify == nil { + return fmt.Errorf("%s: auth plain %s: unknown hash: %s", a.modName, key, parts[0]) + } + return hashVerify(password, parts[1]) +} + +func (a *Auth) ListUsers() ([]string, error) { + tbl, ok := a.table.(module.MutableTable) + if !ok { + return nil, fmt.Errorf("%s: table is not mutable, no management functionality available", a.modName) + } + + l, err := tbl.Keys() + if err != nil { + return nil, fmt.Errorf("%s: list users: %w", a.modName, err) + } + return l, nil +} + +func (a *Auth) CreateUser(username, password string) error { + return a.CreateUserHash(username, password, HashBcrypt, HashOpts{ + BcryptCost: bcrypt.DefaultCost, + }) +} + +func (a *Auth) CreateUserHash(username, password string, hashAlgo string, opts HashOpts) error { + tbl, ok := a.table.(module.MutableTable) + if !ok { + return fmt.Errorf("%s: table is not mutable, no management functionality available", a.modName) + } + + if _, ok := HashCompute[hashAlgo]; !ok { + return fmt.Errorf("%s: unknown hash function: %v", a.modName, hashAlgo) + } + + key, err := precis.UsernameCaseMapped.CompareKey(username) + if err != nil { + return fmt.Errorf("%s: create user %s (raw): %w", a.modName, username, err) + } + + _, ok, err = tbl.Lookup(context.TODO(), key) + if err != nil { + return fmt.Errorf("%s: create user %s: %w", a.modName, key, err) + } + if ok { + return fmt.Errorf("%s: credentials for %s already exist", a.modName, key) + } + + hash, err := HashCompute[hashAlgo](opts, password) + if err != nil { + return fmt.Errorf("%s: create user %s: hash generation: %w", a.modName, key, err) + } + + if err := tbl.SetKey(key, hashAlgo+":"+hash); err != nil { + return fmt.Errorf("%s: create user %s: %w", a.modName, key, err) + } + return nil +} + +func (a *Auth) SetUserPassword(username, password string) error { + tbl, ok := a.table.(module.MutableTable) + if !ok { + return fmt.Errorf("%s: table is not mutable, no management functionality available", a.modName) + } + + key, err := precis.UsernameCaseMapped.CompareKey(username) + if err != nil { + return fmt.Errorf("%s: set password %s (raw): %w", a.modName, username, err) + } + + // TODO: Allow to customize hash function. + hash, err := HashCompute[HashBcrypt](HashOpts{ + BcryptCost: bcrypt.DefaultCost, + }, password) + if err != nil { + return fmt.Errorf("%s: set password %s: hash generation: %w", a.modName, key, err) + } + + if err := tbl.SetKey(key, "bcrypt:"+hash); err != nil { + return fmt.Errorf("%s: set password %s: %w", a.modName, key, err) + } + return nil +} + +func (a *Auth) DeleteUser(username string) error { + tbl, ok := a.table.(module.MutableTable) + if !ok { + return fmt.Errorf("%s: table is not mutable, no management functionality available", a.modName) + } + + key, err := precis.UsernameCaseMapped.CompareKey(username) + if err != nil { + return fmt.Errorf("%s: del user %s (raw): %w", a.modName, username, err) + } + + if err := tbl.RemoveKey(key); err != nil { + return fmt.Errorf("%s: del user %s: %w", a.modName, key, err) + } + return nil +} + +func init() { + module.Register("auth.pass_table", New) +} diff --git a/internal/auth/pass_table/table_test.go b/internal/auth/pass_table/table_test.go new file mode 100644 index 0000000..666e2b6 --- /dev/null +++ b/internal/auth/pass_table/table_test.go @@ -0,0 +1,64 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package pass_table + +import ( + "testing" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestAuth_AuthPlain(t *testing.T) { + addSHA256() + + mod, err := New("pass_table", "", nil, []string{"dummy"}) + if err != nil { + t.Fatal(err) + } + err = mod.Init(config.NewMap(nil, config.Node{ + Children: []config.Node{}, + })) + if err != nil { + t.Fatal(err) + } + a := mod.(*Auth) + a.table = testutils.Table{ + M: map[string]string{ + "foxcpp": "sha256:U0FMVA==:8PDRAgaUqaLSk34WpYniXjaBgGM93Lc6iF4pw2slthw=", + "not-foxcpp": "bcrypt:$2y$10$4tEJtJ6dApmhETg8tJ4WHOeMtmYXQwmHDKIyfg09Bw1F/smhLjlaa", + "not-foxcpp-2": "argon2:1:8:1:U0FBQUFBTFQ=:KHUshl3DcpHR3AoVd28ZeBGmZ1Fj1gwJgNn98Ia8DAvGHqI0BvFOMJPxtaAfO8F+qomm2O3h0P0yV50QGwXI/Q==", + }, + } + + check := func(user, pass string, ok bool) { + t.Helper() + + err := a.AuthPlain(user, pass) + if (err == nil) != ok { + t.Errorf("ok=%v, err: %v", ok, err) + } + } + + check("foxcpp", "password", true) + check("foxcpp", "different-password", false) + check("not-foxcpp", "password", true) + check("not-foxcpp", "different-password", false) + check("not-foxcpp-2", "password", true) +} diff --git a/internal/auth/plain_separate/plain_separate.go b/internal/auth/plain_separate/plain_separate.go new file mode 100644 index 0000000..893d1f0 --- /dev/null +++ b/internal/auth/plain_separate/plain_separate.go @@ -0,0 +1,145 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package plain_separate + +import ( + "context" + "errors" + "fmt" + + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +type Auth struct { + modName string + instName string + + userTbls []module.Table + passwd []module.PlainAuth + + onlyFirstID bool + + Log log.Logger +} + +func NewAuth(modName, instName string, _, inlinargs []string) (module.Module, error) { + a := &Auth{ + modName: modName, + instName: instName, + onlyFirstID: false, + Log: log.Logger{Name: modName}, + } + + if len(inlinargs) != 0 { + return nil, errors.New("plain_separate: inline arguments are not used") + } + + return a, nil +} + +func (a *Auth) Name() string { + return a.modName +} + +func (a *Auth) InstanceName() string { + return a.instName +} + +func (a *Auth) Init(cfg *config.Map) error { + cfg.Bool("debug", false, false, &a.Log.Debug) + cfg.Callback("user", func(m *config.Map, node config.Node) error { + var tbl module.Table + err := modconfig.ModuleFromNode("table", node.Args, node, m.Globals, &tbl) + if err != nil { + return err + } + + a.userTbls = append(a.userTbls, tbl) + return nil + }) + cfg.Callback("pass", func(m *config.Map, node config.Node) error { + var auth module.PlainAuth + err := modconfig.ModuleFromNode("auth", node.Args, node, m.Globals, &auth) + if err != nil { + return err + } + + a.passwd = append(a.passwd, auth) + return nil + }) + + if _, err := cfg.Process(); err != nil { + return err + } + + return nil +} + +func (a *Auth) Lookup(ctx context.Context, username string) (string, bool, error) { + ok := len(a.userTbls) == 0 + for _, tbl := range a.userTbls { + _, tblOk, err := tbl.Lookup(ctx, username) + if err != nil { + return "", false, fmt.Errorf("plain_separate: underlying table error: %w", err) + } + if tblOk { + ok = true + break + } + } + if !ok { + return "", false, nil + } + return "", true, nil +} + +func (a *Auth) AuthPlain(username, password string) error { + ok := len(a.userTbls) == 0 + for _, tbl := range a.userTbls { + _, tblOk, err := tbl.Lookup(context.TODO(), username) + if err != nil { + return err + } + if tblOk { + ok = true + break + } + } + if !ok { + return errors.New("user not found in tables") + } + + var lastErr error + for _, p := range a.passwd { + if err := p.AuthPlain(username, password); err != nil { + lastErr = err + continue + } + + return nil + } + return lastErr +} + +func init() { + module.Register("auth.plain_separate", NewAuth) +} diff --git a/internal/auth/plain_separate/plain_separate_test.go b/internal/auth/plain_separate/plain_separate_test.go new file mode 100644 index 0000000..e5365cc --- /dev/null +++ b/internal/auth/plain_separate/plain_separate_test.go @@ -0,0 +1,155 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package plain_separate + +import ( + "context" + "errors" + "testing" + + "github.com/emersion/go-sasl" + "github.com/foxcpp/maddy/framework/module" +) + +type mockAuth struct { + db map[string]bool +} + +func (mockAuth) SASLMechanisms() []string { + return []string{sasl.Plain, sasl.Login} +} + +func (m mockAuth) AuthPlain(username, _ string) error { + ok := m.db[username] + if !ok { + return errors.New("invalid creds") + } + return nil +} + +type mockTable struct { + db map[string]string +} + +func (m mockTable) Lookup(_ context.Context, a string) (string, bool, error) { + b, ok := m.db[a] + return b, ok, nil +} + +func TestPlainSplit_NoUser(t *testing.T) { + a := Auth{ + passwd: []module.PlainAuth{ + mockAuth{ + db: map[string]bool{ + "user1": true, + }, + }, + }, + } + + err := a.AuthPlain("user1", "aaa") + if err != nil { + t.Fatal("Unexpected error:", err) + } +} + +func TestPlainSplit_NoUser_MultiPass(t *testing.T) { + a := Auth{ + passwd: []module.PlainAuth{ + mockAuth{ + db: map[string]bool{ + "user2": true, + }, + }, + mockAuth{ + db: map[string]bool{ + "user1": true, + }, + }, + }, + } + + err := a.AuthPlain("user1", "aaa") + if err != nil { + t.Fatal("Unexpected error:", err) + } +} + +func TestPlainSplit_UserPass(t *testing.T) { + a := Auth{ + userTbls: []module.Table{ + mockTable{ + db: map[string]string{ + "user1": "", + }, + }, + }, + passwd: []module.PlainAuth{ + mockAuth{ + db: map[string]bool{ + "user2": true, + }, + }, + mockAuth{ + db: map[string]bool{ + "user1": true, + }, + }, + }, + } + + err := a.AuthPlain("user1", "aaa") + if err != nil { + t.Fatal("Unexpected error:", err) + } +} + +func TestPlainSplit_MultiUser_Pass(t *testing.T) { + a := Auth{ + userTbls: []module.Table{ + mockTable{ + db: map[string]string{ + "userWH": "", + }, + }, + mockTable{ + db: map[string]string{ + "user1": "", + }, + }, + }, + passwd: []module.PlainAuth{ + mockAuth{ + db: map[string]bool{ + "user2": true, + }, + }, + mockAuth{ + db: map[string]bool{ + "user1": true, + }, + }, + }, + } + + err := a.AuthPlain("user1", "aaa") + if err != nil { + t.Fatal("Unexpected error:", err) + } +} diff --git a/internal/auth/sasl.go b/internal/auth/sasl.go new file mode 100644 index 0000000..8510052 --- /dev/null +++ b/internal/auth/sasl.go @@ -0,0 +1,207 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package auth + +import ( + "context" + "errors" + "fmt" + "net" + + "github.com/emersion/go-sasl" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth/sasllogin" + "github.com/foxcpp/maddy/internal/authz" +) + +var ( + ErrUnsupportedMech = errors.New("Unsupported SASL mechanism") + ErrInvalidAuthCred = errors.New("auth: invalid credentials") +) + +// SASLAuth is a wrapper that initializes sasl.Server using authenticators that +// call maddy module objects. +// +// It also handles username translation using auth_map and auth_map_normalize +// (AuthMap and AuthMapNormalize should be set). +// +// It supports reporting of multiple authorization identities so multiple +// accounts can be associated with a single set of credentials. +type SASLAuth struct { + Log log.Logger + OnlyFirstID bool + EnableLogin bool + + AuthMap module.Table + AuthNormalize authz.NormalizeFunc + + Plain []module.PlainAuth +} + +func (s *SASLAuth) SASLMechanisms() []string { + var mechs []string + + if len(s.Plain) != 0 { + mechs = append(mechs, sasl.Plain) + if s.EnableLogin { + mechs = append(mechs, sasl.Login) + } + } + + return mechs +} + +func (s *SASLAuth) usernameForAuth(ctx context.Context, saslUsername string) (string, error) { + if s.AuthNormalize != nil { + var err error + saslUsername, err = s.AuthNormalize(saslUsername) + if err != nil { + return "", err + } + } + + if s.AuthMap == nil { + return saslUsername, nil + } + + mapped, ok, err := s.AuthMap.Lookup(ctx, saslUsername) + if err != nil { + return "", err + } + if !ok { + return "", ErrInvalidAuthCred + } + + if saslUsername != mapped { + s.Log.DebugMsg("using mapped username for authentication", "username", saslUsername, "mapped_username", mapped) + } + + return mapped, nil +} + +func (s *SASLAuth) AuthPlain(username, password string) error { + if len(s.Plain) == 0 { + return ErrUnsupportedMech + } + + var lastErr error + for _, p := range s.Plain { + mappedUsername, err := s.usernameForAuth(context.TODO(), username) + if err != nil { + return err + } + + s.Log.DebugMsg("attempting authentication", + "mapped_username", mappedUsername, "original_username", username, + "module", p) + + lastErr = p.AuthPlain(mappedUsername, password) + if lastErr == nil { + return nil + } + } + + return fmt.Errorf("no auth. provider accepted creds, last err: %w", lastErr) +} + +type ContextData struct { + // Authentication username. May be different from identity. + Username string + + // Password used for password-based mechanisms. + Password string +} + +// CreateSASL creates the sasl.Server instance for the corresponding mechanism. +func (s *SASLAuth) CreateSASL(mech string, remoteAddr net.Addr, successCb func(identity string, data ContextData) error) sasl.Server { + switch mech { + case sasl.Plain: + return sasl.NewPlainServer(func(identity, username, password string) error { + if identity == "" { + identity = username + } + if identity != username { + return ErrInvalidAuthCred + } + + err := s.AuthPlain(username, password) + if err != nil { + s.Log.Error("authentication failed", err, "username", username, "src_ip", remoteAddr) + return ErrInvalidAuthCred + } + + return successCb(identity, ContextData{ + Username: username, + Password: password, + }) + }) + case sasl.Login: + if !s.EnableLogin { + return FailingSASLServ{Err: ErrUnsupportedMech} + } + + return sasllogin.NewLoginServer(func(username, password string) error { + username, err := s.usernameForAuth(context.Background(), username) + if err != nil { + return err + } + + err = s.AuthPlain(username, password) + if err != nil { + s.Log.Error("authentication failed", err, "username", username, "src_ip", remoteAddr) + return ErrInvalidAuthCred + } + + return successCb(username, ContextData{ + Username: username, + Password: password, + }) + }) + } + return FailingSASLServ{Err: ErrUnsupportedMech} +} + +// AddProvider adds the SASL authentication provider to its mapping by parsing +// the 'auth' configuration directive. +func (s *SASLAuth) AddProvider(m *config.Map, node config.Node) error { + var any interface{} + if err := modconfig.ModuleFromNode("auth", node.Args, node, m.Globals, &any); err != nil { + return err + } + + hasAny := false + if plainAuth, ok := any.(module.PlainAuth); ok { + s.Plain = append(s.Plain, plainAuth) + hasAny = true + } + + if !hasAny { + return config.NodeErr(node, "auth: specified module does not provide any SASL mechanism") + } + return nil +} + +type FailingSASLServ struct{ Err error } + +func (s FailingSASLServ) Next([]byte) ([]byte, bool, error) { + return nil, true, s.Err +} diff --git a/internal/auth/sasl_test.go b/internal/auth/sasl_test.go new file mode 100644 index 0000000..a59cfc7 --- /dev/null +++ b/internal/auth/sasl_test.go @@ -0,0 +1,89 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package auth + +import ( + "errors" + "net" + "testing" + + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +type mockAuth struct { + db map[string]bool +} + +func (m mockAuth) AuthPlain(username, _ string) error { + ok := m.db[username] + if !ok { + return errors.New("invalid creds") + } + return nil +} + +func TestCreateSASL(t *testing.T) { + a := SASLAuth{ + Log: testutils.Logger(t, "saslauth"), + Plain: []module.PlainAuth{ + &mockAuth{ + db: map[string]bool{ + "user1": true, + }, + }, + }, + } + + t.Run("XWHATEVER", func(t *testing.T) { + srv := a.CreateSASL("XWHATEVER", &net.TCPAddr{}, func(string, ContextData) error { return nil }) + _, _, err := srv.Next([]byte("")) + if err == nil { + t.Error("No error for XWHATEVER use") + } + }) + + t.Run("PLAIN", func(t *testing.T) { + srv := a.CreateSASL("PLAIN", &net.TCPAddr{}, func(id string, data ContextData) error { + if id != "user1" { + t.Fatal("Wrong auth. identities passed to callback:", id) + } + return nil + }) + + _, _, err := srv.Next([]byte("\x00user1\x00aa")) + if err != nil { + t.Error("Unexpected error:", err) + } + }) + + t.Run("PLAIN with authorization identity", func(t *testing.T) { + srv := a.CreateSASL("PLAIN", &net.TCPAddr{}, func(id string, data ContextData) error { + if id != "user1" { + t.Fatal("Wrong authorization identity passed:", id) + } + return nil + }) + + _, _, err := srv.Next([]byte("user1\x00user1\x00aa")) + if err != nil { + t.Error("Unexpected error:", err) + } + }) +} diff --git a/internal/auth/sasllogin/sasllogin.go b/internal/auth/sasllogin/sasllogin.go new file mode 100644 index 0000000..fac5026 --- /dev/null +++ b/internal/auth/sasllogin/sasllogin.go @@ -0,0 +1,54 @@ +package sasllogin + +import "github.com/emersion/go-sasl" + +// Copy-pasted from old emersion/go-sasl version + +// Authenticates users with an username and a password. +type LoginAuthenticator func(username, password string) error +type loginState int + +const ( + loginNotStarted loginState = iota + loginWaitingUsername + loginWaitingPassword +) + +type loginServer struct { + state loginState + username, password string + authenticate LoginAuthenticator +} + +// A server implementation of the LOGIN authentication mechanism, as described +// in https://tools.ietf.org/html/draft-murchison-sasl-login-00. +// +// LOGIN is obsolete and should only be enabled for legacy clients that cannot +// be updated to use PLAIN. +func NewLoginServer(authenticator LoginAuthenticator) sasl.Server { + return &loginServer{authenticate: authenticator} +} + +func (a *loginServer) Next(response []byte) (challenge []byte, done bool, err error) { + switch a.state { + case loginNotStarted: + // Check for initial response field, as per RFC4422 section 3 + if response == nil { + challenge = []byte("Username:") + break + } + a.state++ + fallthrough + case loginWaitingUsername: + a.username = string(response) + challenge = []byte("Password:") + case loginWaitingPassword: + a.password = string(response) + err = a.authenticate(a.username, a.password) + done = true + default: + err = sasl.ErrUnexpectedClientResponse + } + a.state++ + return +} diff --git a/internal/auth/shadow/module.go b/internal/auth/shadow/module.go new file mode 100644 index 0000000..92307bf --- /dev/null +++ b/internal/auth/shadow/module.go @@ -0,0 +1,138 @@ +//go:build !windows +// +build !windows + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package shadow + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth/external" +) + +type Auth struct { + instName string + useHelper bool + helperPath string + + Log log.Logger +} + +func New(modName, instName string, _, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, errors.New("shadow: inline arguments are not used") + } + return &Auth{ + instName: instName, + Log: log.Logger{Name: modName}, + }, nil +} + +func (a *Auth) Name() string { + return "shadow" +} + +func (a *Auth) InstanceName() string { + return a.instName +} + +func (a *Auth) Init(cfg *config.Map) error { + cfg.Bool("debug", true, false, &a.Log.Debug) + cfg.Bool("use_helper", false, false, &a.useHelper) + if _, err := cfg.Process(); err != nil { + return err + } + + if a.useHelper { + a.helperPath = filepath.Join(config.LibexecDirectory, "maddy-shadow-helper") + if _, err := os.Stat(a.helperPath); err != nil { + return fmt.Errorf("shadow: no helper binary (maddy-shadow-helper) found in %s", config.LibexecDirectory) + } + } else { + f, err := os.Open("/etc/shadow") + if err != nil { + if os.IsPermission(err) { + return fmt.Errorf("shadow: can't read /etc/shadow due to permission error, use helper binary or run maddy as a privileged user") + } + return fmt.Errorf("shadow: can't read /etc/shadow: %v", err) + } + f.Close() + } + + return nil +} + +func (a *Auth) Lookup(username string) (string, bool, error) { + if a.useHelper { + return "", false, fmt.Errorf("shadow: table lookup are not possible when using a helper") + } + + ent, err := Lookup(username) + if err != nil { + if errors.Is(err, ErrNoSuchUser) { + return "", false, nil + } + return "", false, err + } + + if !ent.IsAccountValid() { + return "", false, nil + } + + return "", true, nil +} + +func (a *Auth) AuthPlain(username, password string) error { + if a.useHelper { + return external.AuthUsingHelper(a.helperPath, username, password) + } + + ent, err := Lookup(username) + if err != nil { + return err + } + + if !ent.IsAccountValid() { + return fmt.Errorf("shadow: account is expired") + } + + if !ent.IsPasswordValid() { + return fmt.Errorf("shadow: password is expired") + } + + if err := ent.VerifyPassword(password); err != nil { + if errors.Is(err, ErrWrongPassword) { + return module.ErrUnknownCredentials + } + return err + } + + return nil +} + +func init() { + module.Register("auth.shadow", New) +} diff --git a/internal/auth/shadow/read.go b/internal/auth/shadow/read.go new file mode 100644 index 0000000..7059aa0 --- /dev/null +++ b/internal/auth/shadow/read.go @@ -0,0 +1,100 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package shadow + +import ( + "bufio" + "errors" + "fmt" + "os" + "strconv" + "strings" +) + +var ( + ErrNoSuchUser = errors.New("shadow: user entry is not present in database") + ErrWrongPassword = errors.New("shadow: wrong password") +) + +// Read reads system shadow passwords database and returns all entires in it. +func Read() ([]Entry, error) { + f, err := os.Open("/etc/shadow") + if err != nil { + return nil, err + } + scnr := bufio.NewScanner(f) + + var res []Entry + for scnr.Scan() { + ent, err := parseEntry(scnr.Text()) + if err != nil { + return res, err + } + + res = append(res, *ent) + } + if err := scnr.Err(); err != nil { + return res, err + } + return res, nil +} + +func parseEntry(line string) (*Entry, error) { + parts := strings.Split(line, ":") + if len(parts) != 9 { + return nil, errors.New("read: malformed entry") + } + + res := &Entry{ + Name: parts[0], + Pass: parts[1], + } + + for i, value := range [...]*int{ + &res.LastChange, &res.MinPassAge, &res.MaxPassAge, + &res.WarnPeriod, &res.InactivityPeriod, &res.AcctExpiry, &res.Flags, + } { + if parts[2+i] == "" { + *value = -1 + } else { + var err error + *value, err = strconv.Atoi(parts[2+i]) + if err != nil { + return nil, fmt.Errorf("read: invalid value for field %d", 2+i) + } + } + } + + return res, nil +} + +func Lookup(name string) (*Entry, error) { + entries, err := Read() + if err != nil { + return nil, err + } + + for _, entry := range entries { + if entry.Name == name { + return &entry, nil + } + } + + return nil, ErrNoSuchUser +} diff --git a/internal/auth/shadow/shadow.go b/internal/auth/shadow/shadow.go new file mode 100644 index 0000000..eddf6ee --- /dev/null +++ b/internal/auth/shadow/shadow.go @@ -0,0 +1,64 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// shadow package implements utilities for parsing and using shadow password +// database on Unix systems. +package shadow + +type Entry struct { + // User login name. + Name string + + // Hashed user password. + Pass string + + // Days since Jan 1, 1970 password was last changed. + LastChange int + + // The number of days the user will have to wait before she will be allowed to + // change her password again. + // + // -1 if password aging is disabled. + MinPassAge int + + // The number of days after which the user will have to change her password. + // + // -1 is password aging is disabled. + MaxPassAge int + + // The number of days before a password is going to expire (see the maximum + // password age above) during which the user should be warned. + // + // -1 is password aging is disabled. + WarnPeriod int + + // The number of days after a password has expired (see the maximum + // password age above) during which the password should still be accepted. + // + // -1 is password aging is disabled. + InactivityPeriod int + + // The date of expiration of the account, expressed as the number of days + // since Jan 1, 1970. + // + // -1 is account never expires. + AcctExpiry int + + // Unused now. + Flags int +} diff --git a/internal/auth/shadow/verify.go b/internal/auth/shadow/verify.go new file mode 100644 index 0000000..7c983d5 --- /dev/null +++ b/internal/auth/shadow/verify.go @@ -0,0 +1,74 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package shadow + +import ( + "errors" + "fmt" + "time" + + "github.com/GehirnInc/crypt" + _ "github.com/GehirnInc/crypt/sha256_crypt" + _ "github.com/GehirnInc/crypt/sha512_crypt" +) + +const secsInDay = 86400 + +func (e *Entry) IsAccountValid() bool { + if e.AcctExpiry == -1 { + return true + } + + nowDays := int(time.Now().Unix() / secsInDay) + return nowDays < e.AcctExpiry +} + +func (e *Entry) IsPasswordValid() bool { + if e.LastChange == -1 || e.MaxPassAge == -1 || e.InactivityPeriod == -1 { + return true + } + + nowDays := int(time.Now().Unix() / secsInDay) + return nowDays < e.LastChange+e.MaxPassAge+e.InactivityPeriod +} + +func (e *Entry) VerifyPassword(pass string) (err error) { + // Do not permit null and locked passwords. + if e.Pass == "" { + return errors.New("verify: null password") + } + if e.Pass[0] == '!' { + return errors.New("verify: locked password") + } + + // crypt.NewFromHash may panic on unknown hash function. + defer func() { + if rcvr := recover(); rcvr != nil { + err = fmt.Errorf("%v", rcvr) + } + }() + + if err := crypt.NewFromHash(e.Pass).Verify(e.Pass, []byte(pass)); err != nil { + if errors.Is(err, crypt.ErrKeyMismatch) { + return ErrWrongPassword + } + return err + } + return nil +} diff --git a/internal/authz/lookup.go b/internal/authz/lookup.go new file mode 100644 index 0000000..f19c3f8 --- /dev/null +++ b/internal/authz/lookup.go @@ -0,0 +1,44 @@ +package authz + +import ( + "context" + "fmt" + + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/module" +) + +func AuthorizeEmailUse(ctx context.Context, username string, addrs []string, mapping module.Table) (bool, error) { + var validEmails []string + + if multi, ok := mapping.(module.MultiTable); ok { + var err error + validEmails, err = multi.LookupMulti(ctx, username) + if err != nil { + return false, fmt.Errorf("authz: %w", err) + } + } else { + validEmail, ok, err := mapping.Lookup(ctx, username) + if err != nil { + return false, fmt.Errorf("authz: %w", err) + } + if ok { + validEmails = []string{validEmail} + } + } + + for _, addr := range addrs { + _, domain, err := address.Split(addr) + if err != nil { + return false, fmt.Errorf("authz: %w", err) + } + + for _, ent := range validEmails { + if ent == domain || ent == "*" || ent == addr { + return true, nil + } + } + } + + return false, nil +} diff --git a/internal/authz/normalization.go b/internal/authz/normalization.go new file mode 100644 index 0000000..99c46d0 --- /dev/null +++ b/internal/authz/normalization.go @@ -0,0 +1,37 @@ +package authz + +import ( + "strings" + + "github.com/foxcpp/maddy/framework/address" + "golang.org/x/text/secure/precis" +) + +type NormalizeFunc func(string) (string, error) + +func NormalizeNoop(s string) (string, error) { + return s, nil +} + +// NormalizeAuto applies address.PRECISFold to valid emails and +// plain UsernameCaseMapped profile to other strings. +func NormalizeAuto(s string) (string, error) { + if address.Valid(s) { + return address.PRECISFold(s) + } + return precis.UsernameCaseMapped.CompareKey(s) +} + +// NormalizeFuncs defines configurable normalization functions to be used +// in authentication and authorization routines. +var NormalizeFuncs = map[string]NormalizeFunc{ + "auto": NormalizeAuto, + "precis_casefold_email": address.PRECISFold, + "precis_casefold": precis.UsernameCaseMapped.CompareKey, + "precis_email": address.PRECIS, + "precis": precis.UsernameCasePreserved.CompareKey, + "casefold": func(s string) (string, error) { + return strings.ToLower(s), nil + }, + "noop": NormalizeNoop, +} diff --git a/internal/check/authorize_sender/authorize_sender.go b/internal/check/authorize_sender/authorize_sender.go new file mode 100644 index 0000000..6827add --- /dev/null +++ b/internal/check/authorize_sender/authorize_sender.go @@ -0,0 +1,305 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package authorize_sender + +import ( + "context" + "net/mail" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/authz" + "github.com/foxcpp/maddy/internal/table" + "github.com/foxcpp/maddy/internal/target" +) + +const modName = "check.authorize_sender" + +type Check struct { + instName string + log log.Logger + + checkHeader bool + emailPrepare module.Table + userToEmail module.Table + + unauthAction modconfig.FailAction + noMatchAction modconfig.FailAction + errAction modconfig.FailAction + + fromNorm authz.NormalizeFunc + authNorm authz.NormalizeFunc +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + return &Check{ + instName: instName, + }, nil +} + +func (c *Check) Name() string { + return modName +} + +func (c *Check) InstanceName() string { + return c.instName +} + +func (c *Check) Init(cfg *config.Map) error { + cfg.Bool("debug", true, false, &c.log.Debug) + + cfg.Bool("check_header", false, true, &c.checkHeader) + + cfg.Custom("prepare_email", false, false, func() (interface{}, error) { + return &table.Identity{}, nil + }, modconfig.TableDirective, &c.emailPrepare) + cfg.Custom("user_to_email", false, false, func() (interface{}, error) { + return &table.Identity{}, nil + }, modconfig.TableDirective, &c.userToEmail) + + cfg.Custom("unauth_action", false, false, func() (interface{}, error) { + return modconfig.FailAction{Reject: true}, nil + }, modconfig.FailActionDirective, &c.unauthAction) + cfg.Custom("no_match_action", false, false, func() (interface{}, error) { + return modconfig.FailAction{Reject: true}, nil + }, modconfig.FailActionDirective, &c.noMatchAction) + cfg.Custom("err_action", false, false, func() (interface{}, error) { + return modconfig.FailAction{Reject: true}, nil + }, modconfig.FailActionDirective, &c.errAction) + + config.EnumMapped(cfg, "auth_normalize", true, false, authz.NormalizeFuncs, authz.NormalizeAuto, + &c.authNorm) + config.EnumMapped(cfg, "from_normalize", true, false, authz.NormalizeFuncs, authz.NormalizeAuto, + &c.fromNorm) + + if _, err := cfg.Process(); err != nil { + return err + } + + return nil +} + +type state struct { + c *Check + msgMeta *module.MsgMetadata + log log.Logger +} + +func (c *Check) CheckStateForMsg(_ context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &state{ + c: c, + msgMeta: msgMeta, + log: target.DeliveryLogger(c.log, msgMeta), + }, nil +} + +func (s *state) authzSender(ctx context.Context, authName, email string) module.CheckResult { + if authName == "" { + return s.c.unauthAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 530, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Authentication required", + CheckName: modName, + }}) + } + + fromEmailNorm, err := s.c.fromNorm(email) + if err != nil { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 1, 7}, + Message: "Unable to normalize sender address", + CheckName: modName, + Err: err, + }}) + } + authNameNorm, err := s.c.authNorm(authName) + if err != nil { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 535, + EnhancedCode: exterrors.EnhancedCode{5, 7, 8}, + Message: "Unable to normalize authorization username", + CheckName: modName, + }}) + } + + var preparedEmail []string + var ok bool + s.log.DebugMsg("normalized names", "from", fromEmailNorm, "auth", authNameNorm) + if emailPrepareMulti, isMulti := s.c.emailPrepare.(module.MultiTable); isMulti { + preparedEmail, err = emailPrepareMulti.LookupMulti(ctx, fromEmailNorm) + ok = len(preparedEmail) > 0 + } else { + var preparedEmail_single string + preparedEmail_single, ok, err = s.c.emailPrepare.Lookup(ctx, fromEmailNorm) + preparedEmail = []string{preparedEmail_single} + } + s.log.DebugMsg("authorized emails", "preparedEmail", preparedEmail, "ok", ok) + if err != nil { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 454, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Internal error during policy check", + CheckName: modName, + Err: err, + }}) + } + if !ok { + preparedEmail = []string{fromEmailNorm} + } + + ok, err = authz.AuthorizeEmailUse(ctx, authNameNorm, preparedEmail, s.c.userToEmail) + if err != nil { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 454, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Internal error during policy check", + CheckName: modName, + Err: err, + }}) + } + if !ok { + return s.c.noMatchAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Unauthorized use of sender address", + CheckName: modName, + }}) + } + + return module.CheckResult{} +} + +func (s *state) CheckConnection(_ context.Context) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckSender(ctx context.Context, fromEmail string) module.CheckResult { + if s.msgMeta.Conn == nil { + s.log.Msg("skipping locally generated message") + return module.CheckResult{} + } + authName := s.msgMeta.Conn.AuthUser + + return s.authzSender(ctx, authName, fromEmail) +} + +func (s *state) CheckRcpt(_ context.Context, _ string) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckBody(ctx context.Context, hdr textproto.Header, _ buffer.Buffer) module.CheckResult { + if !s.c.checkHeader { + return module.CheckResult{} + } + if s.msgMeta.Conn == nil { + s.log.Msg("skipping locally generated message") + return module.CheckResult{} + } + authName := s.msgMeta.Conn.AuthUser + + fromHdr := hdr.Get("From") + if fromHdr == "" { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Missing From header", + CheckName: modName, + }}) + } + list, err := mail.ParseAddressList(fromHdr) + if err != nil || len(list) == 0 { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Malformed From header", + CheckName: modName, + Err: err, + }}) + } + fromEmail := list[0].Address + if len(list) > 1 { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Multiple From addresses are not allowed", + CheckName: modName, + Err: err, + }}) + } + + var senderAddr string + if senderHdr := hdr.Get("Sender"); senderHdr != "" { + sender, err := mail.ParseAddress(senderHdr) + if err != nil { + return s.c.errAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Malformed Sender header", + CheckName: modName, + Err: err, + }}) + } + senderAddr = sender.Address + } + + res := s.authzSender(ctx, authName, fromEmail) + if res.Reason == nil { + return res + } + + if senderAddr != "" && senderAddr != fromEmail { + res = s.authzSender(ctx, authName, senderAddr) + if res.Reason == nil { + return res + } + } + + // Neither matched. + return s.c.noMatchAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Unauthorized use of sender address", + CheckName: modName, + }}) +} + +func (s *state) Close() error { + return nil +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/check/command/command.go b/internal/check/command/command.go new file mode 100644 index 0000000..6093760 --- /dev/null +++ b/internal/check/command/command.go @@ -0,0 +1,401 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package command + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "regexp" + "runtime/trace" + "strconv" + "strings" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +const modName = "check.command" + +type Stage string + +const ( + StageConnection = "conn" + StageSender = "sender" + StageRcpt = "rcpt" + StageBody = "body" +) + +var placeholderRe = regexp.MustCompile(`{[a-zA-Z0-9_]+?}`) + +type Check struct { + instName string + log log.Logger + + stage Stage + actions map[int]modconfig.FailAction + cmd string + cmdArgs []string +} + +func New(modName, instName string, aliases, inlineArgs []string) (module.Module, error) { + c := &Check{ + instName: instName, + actions: map[int]modconfig.FailAction{ + 1: { + Reject: true, + }, + 2: { + Quarantine: true, + }, + }, + } + + if len(inlineArgs) == 0 { + return nil, errors.New("command: at least one argument is required (command name)") + } + + c.cmd = inlineArgs[0] + c.cmdArgs = inlineArgs[1:] + + return c, nil +} + +func (c *Check) Name() string { + return modName +} + +func (c *Check) InstanceName() string { + return c.instName +} + +func (c *Check) Init(cfg *config.Map) error { + // Check whether the inline argument command is usable. + if _, err := exec.LookPath(c.cmd); err != nil { + return fmt.Errorf("command: %w", err) + } + + cfg.Enum("run_on", false, false, + []string{StageConnection, StageSender, StageRcpt, StageBody}, StageBody, + (*string)(&c.stage)) + + cfg.AllowUnknown() + unknown, err := cfg.Process() + if err != nil { + return err + } + + for _, node := range unknown { + switch node.Name { + case "code": + if len(node.Args) < 2 { + return config.NodeErr(node, "at least two arguments are required: ") + } + exitCode, err := strconv.Atoi(node.Args[0]) + if err != nil { + return config.NodeErr(node, "%v", err) + } + action, err := modconfig.ParseActionDirective(node.Args[1:]) + if err != nil { + return config.NodeErr(node, "%v", err) + } + + c.actions[exitCode] = action + default: + return config.NodeErr(node, "unexpected directive: %v", node.Name) + } + } + + return nil +} + +type state struct { + c *Check + msgMeta *module.MsgMetadata + log log.Logger + + mailFrom string + rcpts []string +} + +func (c *Check) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &state{ + c: c, + msgMeta: msgMeta, + log: target.DeliveryLogger(c.log, msgMeta), + }, nil +} + +func (s *state) expandCommand(address string) (string, []string) { + expArgs := make([]string, len(s.c.cmdArgs)) + + for i, arg := range s.c.cmdArgs { + expArgs[i] = placeholderRe.ReplaceAllStringFunc(arg, func(placeholder string) string { + switch placeholder { + case "{auth_user}": + if s.msgMeta.Conn == nil { + return "" + } + return s.msgMeta.Conn.AuthUser + case "{source_ip}": + if s.msgMeta.Conn == nil { + return "" + } + tcpAddr, _ := s.msgMeta.Conn.RemoteAddr.(*net.TCPAddr) + if tcpAddr == nil { + return "" + } + return tcpAddr.IP.String() + case "{source_host}": + if s.msgMeta.Conn == nil { + return "" + } + return s.msgMeta.Conn.Hostname + case "{source_rdns}": + if s.msgMeta.Conn == nil { + return "" + } + valI, err := s.msgMeta.Conn.RDNSName.Get() + if err != nil { + return "" + } + if valI == nil { + return "" + } + return valI.(string) + case "{msg_id}": + return s.msgMeta.ID + case "{sender}": + return s.mailFrom + case "{rcpts}": + return strings.Join(s.rcpts, "\n") + case "{address}": + return address + } + return placeholder + }) + } + + return s.c.cmd, expArgs +} + +func (s *state) run(cmdName string, args []string, stdin io.Reader) module.CheckResult { + cmd := exec.Command(cmdName, args...) + cmd.Stdin = stdin + stdout, err := cmd.StdoutPipe() + if err != nil { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 450, + Message: "Internal server error", + CheckName: "command", + Err: err, + Misc: map[string]interface{}{ + "cmd": cmd.String(), + }, + }, + Reject: true, + } + } + + if err := cmd.Start(); err != nil { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 450, + Message: "Internal server error", + CheckName: "command", + Err: err, + Misc: map[string]interface{}{ + "cmd": cmd.String(), + }, + }, + Reject: true, + } + } + + bufOut := bufio.NewReader(stdout) + hdr, err := textproto.ReadHeader(bufOut) + if err != nil && !errors.Is(err, io.EOF) { + if err := cmd.Process.Signal(os.Interrupt); err != nil { + s.log.Error("failed to kill process", err) + } + + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 450, + Message: "Internal server error", + CheckName: "command", + Err: err, + Misc: map[string]interface{}{ + "cmd": cmd.String(), + }, + }, + Reject: true, + } + } + + res := module.CheckResult{} + res.Header = hdr + + err = cmd.Wait() + if err != nil { + if _, ok := err.(*exec.ExitError); !ok { + // If that's not ExitError, the process may still be running. We do + // not want this. + if err := cmd.Process.Signal(os.Interrupt); err != nil { + s.log.Error("failed to kill process", err) + } + } + return s.errorRes(err, res, cmd.String()) + } + return res +} + +func (s *state) errorRes(err error, res module.CheckResult, cmdLine string) module.CheckResult { + exitErr, ok := err.(*exec.ExitError) + if !ok { + res.Reason = &exterrors.SMTPError{ + Code: 450, + Message: "Internal server error", + CheckName: "command", + Err: err, + Misc: map[string]interface{}{ + "cmd": cmdLine, + }, + } + res.Reject = true + return res + } + + action, ok := s.c.actions[exitErr.ExitCode()] + if !ok { + res.Reason = &exterrors.SMTPError{ + Code: 450, + Message: "Internal server error", + CheckName: "command", + Err: err, + Reason: "unexpected exit code", + Misc: map[string]interface{}{ + "cmd": cmdLine, + "exit_code": exitErr.ExitCode(), + }, + } + res.Reject = true + return res + } + + res.Reason = &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 1}, + Message: "Message rejected for due to a local policy", + CheckName: "command", + Misc: map[string]interface{}{ + "cmd": cmdLine, + "exit_code": exitErr.ExitCode(), + }, + } + + return action.Apply(res) +} + +func (s *state) CheckConnection(ctx context.Context) module.CheckResult { + if s.c.stage != StageConnection { + return module.CheckResult{} + } + + defer trace.StartRegion(ctx, "command/CheckConnection-"+s.c.cmd).End() + + cmdName, cmdArgs := s.expandCommand("") + return s.run(cmdName, cmdArgs, bytes.NewReader(nil)) +} + +func (s *state) CheckSender(ctx context.Context, addr string) module.CheckResult { + s.mailFrom = addr + + if s.c.stage != StageSender { + return module.CheckResult{} + } + + defer trace.StartRegion(ctx, "command/CheckSender"+s.c.cmd).End() + + cmdName, cmdArgs := s.expandCommand(addr) + return s.run(cmdName, cmdArgs, bytes.NewReader(nil)) +} + +func (s *state) CheckRcpt(ctx context.Context, addr string) module.CheckResult { + s.rcpts = append(s.rcpts, addr) + + if s.c.stage != StageRcpt { + return module.CheckResult{} + } + defer trace.StartRegion(ctx, "command/CheckRcpt"+s.c.cmd).End() + + cmdName, cmdArgs := s.expandCommand(addr) + return s.run(cmdName, cmdArgs, bytes.NewReader(nil)) +} + +func (s *state) CheckBody(ctx context.Context, hdr textproto.Header, body buffer.Buffer) module.CheckResult { + if s.c.stage != StageBody { + return module.CheckResult{} + } + + defer trace.StartRegion(ctx, "command/CheckBody"+s.c.cmd).End() + + cmdName, cmdArgs := s.expandCommand("") + + var buf bytes.Buffer + _ = textproto.WriteHeader(&buf, hdr) + bR, err := body.Open() + if err != nil { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 450, + Message: "Internal server error", + CheckName: "command", + Err: err, + Misc: map[string]interface{}{ + "cmd": cmdName + " " + strings.Join(cmdArgs, " "), + }, + }, + Reject: true, + } + } + + return s.run(cmdName, cmdArgs, io.MultiReader(bytes.NewReader(buf.Bytes()), bR)) +} + +func (s *state) Close() error { + return nil +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/check/dkim/dkim.go b/internal/check/dkim/dkim.go new file mode 100644 index 0000000..563fc5b --- /dev/null +++ b/internal/check/dkim/dkim.go @@ -0,0 +1,272 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dkim + +import ( + "bytes" + "context" + "errors" + "io" + nettextproto "net/textproto" + "runtime/trace" + "strings" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/emersion/go-msgauth/dkim" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +type Check struct { + instName string + log log.Logger + + requiredFields map[string]struct{} + brokenSigAction modconfig.FailAction + noSigAction modconfig.FailAction + failOpen bool + + resolver dns.Resolver +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, errors.New("check.dkim: inline arguments are not used") + } + return &Check{ + instName: instName, + log: log.Logger{Name: "check.dkim"}, + resolver: dns.DefaultResolver(), + }, nil +} + +func (c *Check) Init(cfg *config.Map) error { + var requiredFields []string + + cfg.Bool("debug", true, false, &c.log.Debug) + cfg.StringList("required_fields", false, false, []string{"From", "Subject"}, &requiredFields) + cfg.Bool("fail_open", false, false, &c.failOpen) + cfg.Custom("broken_sig_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.brokenSigAction) + cfg.Custom("no_sig_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.noSigAction) + _, err := cfg.Process() + if err != nil { + return err + } + + c.requiredFields = make(map[string]struct{}) + for _, field := range requiredFields { + c.requiredFields[nettextproto.CanonicalMIMEHeaderKey(field)] = struct{}{} + } + + return nil +} + +func (c *Check) Name() string { + return "check.dkim" +} + +func (c *Check) InstanceName() string { + return c.instName +} + +type dkimCheckState struct { + c *Check + msgMeta *module.MsgMetadata + log log.Logger +} + +func (d *dkimCheckState) CheckConnection(ctx context.Context) module.CheckResult { + return module.CheckResult{} +} + +func (d *dkimCheckState) CheckSender(ctx context.Context, mailFrom string) module.CheckResult { + return module.CheckResult{} +} + +func (d *dkimCheckState) CheckRcpt(ctx context.Context, rcptTo string) module.CheckResult { + return module.CheckResult{} +} + +func (d *dkimCheckState) CheckBody(ctx context.Context, header textproto.Header, body buffer.Buffer) module.CheckResult { + defer trace.StartRegion(ctx, "check.dkim/CheckBody").End() + + if !header.Has("DKIM-Signature") { + if d.c.noSigAction.Reject || d.c.noSigAction.Quarantine { + d.log.Printf("no signatures present") + } else { + d.log.Debugf("no signatures present") + } + return d.c.noSigAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 20}, + Message: "No DKIM signatures", + CheckName: "check.dkim", + }, + AuthResult: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultNone, + }, + }, + }) + } + + b := bytes.Buffer{} + _ = textproto.WriteHeader(&b, header) + bodyRdr, err := body.Open() + if err != nil { + return module.CheckResult{ + Reject: true, + Reason: exterrors.WithTemporary( + exterrors.WithFields(err, map[string]interface{}{ + "check": "check.dkim", + "smtp_msg": "Internal I/O error", + }), + true, + ), + } + } + + verifications, err := dkim.VerifyWithOptions(io.MultiReader(&b, bodyRdr), &dkim.VerifyOptions{ + LookupTXT: func(domain string) ([]string, error) { + return d.c.resolver.LookupTXT(ctx, domain) + }, + }) + if err != nil { + return module.CheckResult{ + Reject: true, + Reason: exterrors.WithTemporary( + exterrors.WithFields(err, map[string]interface{}{ + "check": "check.dkim", + "smtp_msg": "Internal error during policy check", + }), + true, + ), + } + } + + goodSigs := false + + res := module.CheckResult{AuthResult: make([]authres.Result, 0, len(verifications))} + for _, verif := range verifications { + val := authres.ResultValue(authres.ResultPass) + reason := "" + if verif.Err != nil { + val = authres.ResultFail + + reason = strings.TrimPrefix(verif.Err.Error(), "dkim: ") + if !d.c.brokenSigAction.Reject || !d.c.brokenSigAction.Quarantine { + d.log.DebugMsg("bad signature", "domain", verif.Domain, "identifier", verif.Identifier) + } + if dkim.IsPermFail(verif.Err) { + val = authres.ResultPermError + } + if dkim.IsTempFail(verif.Err) { + if !d.c.failOpen { + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 421, + EnhancedCode: exterrors.EnhancedCode{4, 7, 20}, + Message: "Temporary error during DKIM verification", + CheckName: "check.dkim", + Err: verif.Err, + }, + } + } + val = authres.ResultTempError + } + + res.AuthResult = append(res.AuthResult, &authres.DKIMResult{ + Value: val, + Reason: reason, + Domain: verif.Domain, + Identifier: verif.Identifier, + }) + continue + } + + signedFields := make(map[string]struct{}, len(verif.HeaderKeys)) + for _, field := range verif.HeaderKeys { + signedFields[nettextproto.CanonicalMIMEHeaderKey(field)] = struct{}{} + } + for field := range d.c.requiredFields { + if _, ok := signedFields[field]; !ok { + val = authres.ResultPermError + reason = "some header fields are not signed" + } + } + + if val == authres.ResultPass { + goodSigs = true + d.log.DebugMsg("good signature", "domain", verif.Domain, "identifier", verif.Identifier) + } + + res.AuthResult = append(res.AuthResult, &authres.DKIMResult{ + Value: val, + Reason: reason, + Domain: verif.Domain, + Identifier: verif.Identifier, + }) + } + + if !goodSigs { + res.Reason = &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 20}, + Message: "No passing DKIM signatures", + CheckName: "check.dkim", + } + return d.c.brokenSigAction.Apply(res) + } + return res +} + +func (d *dkimCheckState) Name() string { + return "check.dkim" +} + +func (d *dkimCheckState) Close() error { + return nil +} + +func (c *Check) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &dkimCheckState{ + c: c, + msgMeta: msgMeta, + log: target.DeliveryLogger(c.log, msgMeta), + }, nil +} + +func init() { + module.Register("check.dkim", New) +} diff --git a/internal/check/dkim/dkim_test.go b/internal/check/dkim/dkim_test.go new file mode 100644 index 0000000..020d50f --- /dev/null +++ b/internal/check/dkim/dkim_test.go @@ -0,0 +1,364 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dkim + +import ( + "context" + "errors" + "net" + "testing" + + "github.com/emersion/go-msgauth/authres" + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +const unsignedMailString = `From: Joe SixPack +To: Suzie Q +Subject: Is dinner ready? +Date: Fri, 11 Jul 2003 21:00:37 -0700 (PDT) +Message-ID: <20030712040037.46341.5F8J@football.example.com> + +Hi. + +We lost the game. Are you hungry yet? + +Joe. +` + +const dnsPublicKey = "v=DKIM1; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQ" + + "KBgQDwIRP/UC3SBsEmGqZ9ZJW3/DkMoGeLnQg1fWn7/zYt" + + "IxN2SnFCjxOCKG9v3b4jYfcTNh5ijSsq631uBItLa7od+v" + + "/RtdC2UzJ1lWT947qR+Rcac2gbto/NMqJ0fzfVjH4OuKhi" + + "tdY9tf6mcwGjaNBcWToIMmPSPDdQPNUYckcQ2QIDAQAB" + +var testZones = map[string]mockdns.Zone{ + "brisbane._domainkey.example.com.": { + TXT: []string{dnsPublicKey}, + }, +} + +const verifiedMailString = `DKIM-Signature: v=1; a=rsa-sha256; s=brisbane; d=example.com; + c=simple/simple; q=dns/txt; i=joe@football.example.com; + h=Received : From : To : Subject : Date : Message-ID; + bh=2jUSOH9NhtVGCQWNr9BrIAPreKQjO6Sn7XIkfJVOzv8=; + b=AuUoFEfDxTDkHlLXSZEpZj79LICEps6eda7W3deTVFOk4yAUoqOB + 4nujc7YopdG5dWLSdNg6xNAZpOPr+kHxt1IrE+NahM6L/LbvaHut + KVdkLLkpVaVVQPzeRDI009SO2Il5Lu7rDNH6mZckBdrIx0orEtZV + 4bmp/YzhwvcubU4=; +Received: from client1.football.example.com [192.0.2.1] + by submitserver.example.com with SUBMISSION; + Fri, 11 Jul 2003 21:01:54 -0700 (PDT) +From: Joe SixPack +To: Suzie Q +Subject: Is dinner ready? +Date: Fri, 11 Jul 2003 21:00:37 -0700 (PDT) +Message-ID: <20030712040037.46341.5F8J@football.example.com> + +Hi. + +We lost the game. Are you hungry yet? + +Joe. +` + +func testCheck(t *testing.T, zones map[string]mockdns.Zone, cfg []config.Node) *Check { + t.Helper() + mod, err := New("check.dkim", "", nil, nil) + if err != nil { + t.Fatal(err) + } + check := mod.(*Check) + check.resolver = &mockdns.Resolver{Zones: zones} + check.log = testutils.Logger(t, mod.Name()) + + if err := check.Init(config.NewMap(nil, config.Node{Children: cfg})); err != nil { + t.Fatal(err) + } + + return check +} + +func TestDkimVerify_NoSig(t *testing.T) { + check := testCheck(t, nil, nil) // No zones since this test requires no lookups. + + // Force certain reason so we can assert for it. + check.noSigAction.Reject = true + check.noSigAction.ReasonOverride = &exterrors.SMTPError{Code: 555} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // The usual checking flow. + s, err := check.CheckStateForMsg(ctx, &module.MsgMetadata{ + ID: "test_unsigned", + }) + if err != nil { + t.Fatal(err) + } + s.CheckConnection(ctx) + s.CheckSender(ctx, "joe@football.example.com") + s.CheckRcpt(ctx, "suzie@shopping.example.net") + + hdr, buf := testutils.BodyFromStr(t, unsignedMailString) + result := s.CheckBody(ctx, hdr, buf) + + if result.Reason == nil { + t.Fatal("No check fail reason set, auth. result:", authres.Format("", result.AuthResult)) + } + if result.Reason.(*exterrors.SMTPError).Code != 555 { + t.Fatal("Different fail reason:", result.Reason) + } +} + +func TestDkimVerify_InvalidSig(t *testing.T) { + check := testCheck(t, testZones, nil) + + // Force certain reason so we can assert for it. + check.brokenSigAction.Reject = true + check.brokenSigAction.ReasonOverride = &exterrors.SMTPError{Code: 555} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := check.CheckStateForMsg(ctx, &module.MsgMetadata{ + ID: "test_unsigned", + }) + if err != nil { + t.Fatal(err) + } + + s.CheckConnection(ctx) + s.CheckSender(ctx, "joe@football.example.com") + s.CheckRcpt(ctx, "suzie@shopping.example.net") + + hdr, buf := testutils.BodyFromStr(t, verifiedMailString) + // Mess up the signature. + hdr.Set("From", "nope") + + result := s.CheckBody(ctx, hdr, buf) + + if result.Reason == nil { + t.Fatal("No check fail reason set, auth. result:", authres.Format("", result.AuthResult)) + } + if result.Reason.(*exterrors.SMTPError).Code != 555 { + t.Fatal("Different fail reason:", result.Reason) + } +} + +func TestDkimVerify_ValidSig(t *testing.T) { + check := testCheck(t, testZones, nil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := check.CheckStateForMsg(ctx, &module.MsgMetadata{ + ID: "test_unsigned", + }) + if err != nil { + t.Fatal(err) + } + + s.CheckConnection(ctx) + s.CheckSender(ctx, "joe@football.example.com") + s.CheckRcpt(ctx, "suzie@shopping.example.net") + + hdr, buf := testutils.BodyFromStr(t, verifiedMailString) + + result := s.CheckBody(ctx, hdr, buf) + + if result.Reason != nil { + t.Log(authres.Format("", result.AuthResult)) + t.Fatal("Check fail reason set, auth. result:", result.Reason, exterrors.Fields(result.Reason)) + } +} + +func TestDkimVerify_RequiredFields(t *testing.T) { + check := testCheck(t, testZones, []config.Node{ + { + // Require field that is not covered by the signature. + Name: "required_fields", + Args: []string{"From", "X-Important"}, + }, + }) + + // Force certain reason so we can assert for it. + check.brokenSigAction.Reject = true + check.brokenSigAction.ReasonOverride = &exterrors.SMTPError{Code: 555} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := check.CheckStateForMsg(ctx, &module.MsgMetadata{ + ID: "test_unsigned", + }) + if err != nil { + t.Fatal(err) + } + + s.CheckConnection(ctx) + s.CheckSender(ctx, "joe@football.example.com") + s.CheckRcpt(ctx, "suzie@shopping.example.net") + + hdr, buf := testutils.BodyFromStr(t, verifiedMailString) + + result := s.CheckBody(ctx, hdr, buf) + + if result.Reason == nil { + t.Fatal("No check fail reason set, auth. result:", authres.Format("", result.AuthResult)) + } + if result.Reason.(*exterrors.SMTPError).Code != 555 { + t.Fatal("Different fail reason:", result.Reason) + } +} + +func TestDkimVerify_BufferOpenFail(t *testing.T) { + check := testCheck(t, testZones, nil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := check.CheckStateForMsg(ctx, &module.MsgMetadata{ + ID: "test_unsigned", + }) + if err != nil { + t.Fatal(err) + } + + s.CheckConnection(ctx) + s.CheckSender(ctx, "joe@football.example.com") + s.CheckRcpt(ctx, "suzie@shopping.example.net") + + var buf buffer.Buffer + hdr, buf := testutils.BodyFromStr(t, verifiedMailString) + buf = testutils.FailingBuffer{Blob: buf.(buffer.MemoryBuffer).Slice, OpenError: errors.New("No!")} + + result := s.CheckBody(ctx, hdr, buf) + t.Log("auth. result:", authres.Format("", result.AuthResult)) + + if result.Reason == nil { + t.Fatal("No check fail reason set, auth. result:", authres.Format("", result.AuthResult)) + } +} + +func TestDkimVerify_FailClosed(t *testing.T) { + zones := map[string]mockdns.Zone{ + "brisbane._domainkey.example.com.": { + Err: &net.DNSError{ + Err: "DNS server is not having a great time", + IsTemporary: true, + IsTimeout: true, + }, + }, + } + check := testCheck(t, zones, []config.Node{ + { + Name: "fail_open", + Args: []string{"false"}, + }, + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := check.CheckStateForMsg(ctx, &module.MsgMetadata{ + ID: "test_unsigned", + }) + if err != nil { + t.Fatal(err) + } + + s.CheckConnection(ctx) + s.CheckSender(ctx, "joe@football.example.com") + s.CheckRcpt(ctx, "suzie@shopping.example.net") + + hdr, buf := testutils.BodyFromStr(t, verifiedMailString) + + result := s.CheckBody(ctx, hdr, buf) + t.Log("auth. result:", authres.Format("", result.AuthResult)) + + if result.Reason == nil { + t.Fatal("No check fail reason set, auth. result:", authres.Format("", result.AuthResult)) + } + if !result.Reject { + t.Fatal("No reject requested") + } + if !exterrors.IsTemporary(result.Reason) { + t.Fatal("Fail reason is not marked as temporary:", result.Reason) + } +} + +func TestDkimVerify_FailOpen(t *testing.T) { + zones := map[string]mockdns.Zone{ + "brisbane._domainkey.example.com.": { + Err: &net.DNSError{ + Err: "DNS server is not having a great time", + IsTemporary: true, + IsTimeout: true, + }, + }, + } + check := testCheck(t, zones, []config.Node{ + { + Name: "fail_open", + Args: []string{"true"}, + }, + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s, err := check.CheckStateForMsg(ctx, &module.MsgMetadata{ + ID: "test_unsigned", + }) + if err != nil { + t.Fatal(err) + } + + s.CheckConnection(ctx) + s.CheckSender(ctx, "joe@football.example.com") + s.CheckRcpt(ctx, "suzie@shopping.example.net") + + hdr, buf := testutils.BodyFromStr(t, verifiedMailString) + + result := s.CheckBody(ctx, hdr, buf) + + t.Log("auth. result:", authres.Format("", result.AuthResult)) + if result.Reason == nil { + t.Fatal("No check fail reason set, auth. result:", authres.Format("", result.AuthResult)) + } + if result.Reject { + t.Fatal("Reject requested") + } + if exterrors.IsTemporary(result.Reason) { + t.Fatal("Fail reason is not marked as temporary:", result.Reason) + } + + if len(result.AuthResult) != 1 { + t.Fatal("Wrong amount of auth. result fields:", len(result.AuthResult)) + } + resVal := result.AuthResult[0].(*authres.DKIMResult).Value + if resVal != authres.ResultTempError { + t.Fatal("Result is not temp. error:", resVal) + } +} diff --git a/internal/check/dns/dns.go b/internal/check/dns/dns.go new file mode 100644 index 0000000..80c89ed --- /dev/null +++ b/internal/check/dns/dns.go @@ -0,0 +1,163 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dns + +import ( + "strings" + + "github.com/foxcpp/maddy/framework/address" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/check" +) + +func requireMatchingRDNS(ctx check.StatelessCheckContext) module.CheckResult { + if ctx.MsgMeta.Conn == nil { + ctx.Logger.Msg("locally-generated message, skipping") + return module.CheckResult{} + } + if ctx.MsgMeta.Conn.RDNSName == nil { + ctx.Logger.Msg("rDNS lookup is disabled, skipping") + return module.CheckResult{} + } + + rdnsNameI, err := ctx.MsgMeta.Conn.RDNSName.Get() + if err != nil { + reason, misc := exterrors.UnwrapDNSErr(err) + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: exterrors.SMTPCode(err, 450, 550), + EnhancedCode: exterrors.SMTPEnchCode(err, exterrors.EnhancedCode{0, 7, 25}), + Message: "DNS error during policy check", + CheckName: "require_matching_rdns", + Err: err, + Reason: reason, + Misc: misc, + }, + } + } + if rdnsNameI == nil { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 25}, + Message: "No PTR record found", + CheckName: "require_matching_rdns", + Err: err, + }, + } + } + rdnsName := rdnsNameI.(string) + + srcDomain := strings.TrimSuffix(ctx.MsgMeta.Conn.Hostname, ".") + rdnsName = strings.TrimSuffix(rdnsName, ".") + + if dns.Equal(rdnsName, srcDomain) { + ctx.Logger.Debugf("PTR record %s matches source domain, OK", rdnsName) + return module.CheckResult{} + } + + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 25}, + Message: "rDNS name does not match source hostname", + CheckName: "require_matching_rdns", + }, + } +} + +func requireMXRecord(ctx check.StatelessCheckContext, mailFrom string) module.CheckResult { + if mailFrom == "" { + // Permit null reverse-path for bounces. + return module.CheckResult{} + } + + _, domain, err := address.Split(mailFrom) + if err != nil { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 8}, + Message: "Malformed sender address", + CheckName: "require_mx_record", + }, + } + } + if domain == "" { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 8}, + Message: "No domain part in address", + CheckName: "require_mx_record", + }, + } + } + + srcMx, err := ctx.Resolver.LookupMX(ctx, domain) + if err != nil { + reason, misc := exterrors.UnwrapDNSErr(err) + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: exterrors.SMTPCode(err, 450, 550), + EnhancedCode: exterrors.SMTPEnchCode(err, exterrors.EnhancedCode{0, 7, 0}), + Message: "DNS error during policy check", + CheckName: "require_mx_record", + Err: err, + Reason: reason, + Misc: misc, + }, + } + } + + if len(srcMx) == 0 { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 7, 27}, + Message: "Domain in MAIL FROM does not have any MX records", + CheckName: "require_mx_record", + }, + } + } + + for _, mx := range srcMx { + if mx.Host == "." { + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 7, 27}, + Message: "Domain in MAIL FROM has null MX record", + CheckName: "require_mx_record", + }, + } + } + } + + return module.CheckResult{} +} +func init() { + check.RegisterStatelessCheck("require_matching_rdns", modconfig.FailAction{Quarantine: true}, + requireMatchingRDNS, nil, nil, nil) + check.RegisterStatelessCheck("require_mx_record", modconfig.FailAction{Quarantine: true}, + nil, requireMXRecord, nil, nil) +} diff --git a/internal/check/dns/dns_test.go b/internal/check/dns/dns_test.go new file mode 100644 index 0000000..ca6c348 --- /dev/null +++ b/internal/check/dns/dns_test.go @@ -0,0 +1,116 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dns + +import ( + "net" + "testing" + + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/framework/future" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/check" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestRequireMatchingRDNS(t *testing.T) { + test := func(rdns, srcHost string, fail bool) { + rdnsFut := future.New() + var ptr []string + if rdns != "" { + rdnsFut.Set(rdns, nil) + ptr = []string{rdns} + } else { + rdnsFut.Set(nil, nil) + } + + res := requireMatchingRDNS(check.StatelessCheckContext{ + Resolver: &mockdns.Resolver{ + Zones: map[string]mockdns.Zone{ + "4.3.2.1.in-addr.arpa.": { + PTR: ptr, + }, + }, + }, + MsgMeta: &module.MsgMetadata{ + Conn: &module.ConnState{ + RemoteAddr: &net.TCPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 55555}, + Hostname: srcHost, + RDNSName: rdnsFut, + }, + }, + Logger: testutils.Logger(t, "require_matching_rdns"), + }) + + actualFail := res.Reason != nil + if fail && !actualFail { + t.Errorf("%v, %s: expected failure but check succeeded", rdns, srcHost) + } + if !fail && actualFail { + t.Errorf("%v, %s: unexpected failure", rdns, srcHost) + } + } + + test("", "example.org", true) + test("example.org", "[1.2.3.4]", true) + test("example.org", "[IPv6:beef::1]", true) + test("example.org", "example.org", false) + test("example.org.", "example.org", false) + test("example.org", "example.org.", false) + test("example.org.", "example.org.", false) + test("example.com.", "example.org.", true) +} + +func TestRequireMXRecord(t *testing.T) { + test := func(mailFrom, mxDomain string, mx []net.MX, fail bool) { + res := requireMXRecord(check.StatelessCheckContext{ + Resolver: &mockdns.Resolver{ + Zones: map[string]mockdns.Zone{ + mxDomain + ".": { + MX: mx, + }, + }, + }, + MsgMeta: &module.MsgMetadata{ + Conn: &module.ConnState{ + RemoteAddr: &net.TCPAddr{IP: net.IPv4(1, 2, 3, 4), Port: 55555}, + }, + }, + Logger: testutils.Logger(t, "require_mx_record"), + }, mailFrom) + + actualFail := res.Reason != nil + if fail && !actualFail { + t.Errorf("%v, %v: expected failure but check succeeded", mailFrom, mx) + } + if !fail && actualFail { + t.Errorf("%v, %v: unexpected failure", mailFrom, mx) + } + } + + test("foo@example.org", "example.org", nil, true) + test("foo@example.com", "", nil, true) // NXDOMAIN + test("foo@[1.2.3.4]", "", nil, true) + test("[IPv6:beef::1]", "", nil, true) + test("[IPv6:beef::1]", "", nil, true) + test("foo@example.org", "example.org", []net.MX{{Host: "a.com"}}, false) + test("foo@", "", nil, true) + test("", "", nil, false) // Permit <> for bounces. + test("foo@example.org", "example.org", []net.MX{{Host: "."}}, true) +} diff --git a/internal/check/dnsbl/common.go b/internal/check/dnsbl/common.go new file mode 100644 index 0000000..7b874cb --- /dev/null +++ b/internal/check/dnsbl/common.go @@ -0,0 +1,197 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dnsbl + +import ( + "context" + "net" + "strconv" + "strings" + + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" +) + +type ListedErr struct { + Identity string + List string + Reason string +} + +func (le ListedErr) Fields() map[string]interface{} { + return map[string]interface{}{ + "check": "dnsbl", + "list": le.List, + "listed_identity": le.Identity, + "reason": le.Reason, + "smtp_code": 554, + "smtp_enchcode": exterrors.EnhancedCode{5, 7, 0}, + "smtp_msg": "Client identity listed in the used DNSBL", + } +} + +func (le ListedErr) Error() string { + return le.Identity + " is listed in the used DNSBL" +} + +func checkDomain(ctx context.Context, resolver dns.Resolver, cfg List, domain string) error { + query := domain + "." + cfg.Zone + + addrs, err := resolver.LookupHost(ctx, query) + if err != nil { + if dnsErr, ok := err.(*net.DNSError); ok && dnsErr.IsNotFound { + return nil + } + + return err + } + + if len(addrs) == 0 { + return nil + } + + // Attempt to extract explanation string. + txts, err := resolver.LookupTXT(context.Background(), query) + if err != nil || len(txts) == 0 { + // Not significant, include addresses as reason. Usually they are + // mapped to some predefined 'reasons' by BL. + return ListedErr{ + Identity: domain, + List: cfg.Zone, + Reason: strings.Join(addrs, "; "), + } + } + + // Some BLs provide multiple reasons (meta-BLs such as Spamhaus Zen) so + // don't mangle them by joining with "", instead join with "; ". + + return ListedErr{ + Identity: domain, + List: cfg.Zone, + Reason: strings.Join(txts, "; "), + } +} + +func checkIP(ctx context.Context, resolver dns.Resolver, cfg List, ip net.IP) error { + ipv6 := true + if ipv4 := ip.To4(); ipv4 != nil { + ip = ipv4 + ipv6 = false + } + + if ipv6 && !cfg.ClientIPv6 { + return nil + } + if !ipv6 && !cfg.ClientIPv4 { + return nil + } + + query := queryString(ip) + "." + cfg.Zone + + addrs, err := resolver.LookupIPAddr(ctx, query) + if err != nil { + if dnsErr, ok := err.(*net.DNSError); ok && dnsErr.IsNotFound { + return nil + } + + return err + } + + filteredAddrs := make([]net.IPAddr, 0, len(addrs)) +addrsLoop: + for _, addr := range addrs { + // No responses whitelist configured - permit all. + if len(cfg.Responses) == 0 { + filteredAddrs = append(filteredAddrs, addr) + continue + } + + for _, respNet := range cfg.Responses { + if respNet.Contains(addr.IP) { + filteredAddrs = append(filteredAddrs, addr) + continue addrsLoop + } + } + } + + if len(filteredAddrs) == 0 { + return nil + } + + // Attempt to extract explanation string. + txts, err := resolver.LookupTXT(ctx, query) + if err != nil || len(txts) == 0 { + // Not significant, include addresses as reason. Usually they are + // mapped to some predefined 'reasons' by BL. + + reasonParts := make([]string, 0, len(filteredAddrs)) + for _, addr := range filteredAddrs { + reasonParts = append(reasonParts, addr.IP.String()) + } + + return ListedErr{ + Identity: ip.String(), + List: cfg.Zone, + Reason: strings.Join(reasonParts, "; "), + } + } + + // Some BLs provide multiple reasons (meta-BLs such as Spamhaus Zen) so + // don't mangle them by joining with "", instead join with "; ". + + return ListedErr{ + Identity: ip.String(), + List: cfg.Zone, + Reason: strings.Join(txts, "; "), + } +} + +func queryString(ip net.IP) string { + ipv6 := true + if ipv4 := ip.To4(); ipv4 != nil { + ip = ipv4 + ipv6 = false + } + + res := strings.Builder{} + if ipv6 { + res.Grow(63) // 0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 + } else { + res.Grow(15) // 000.000.000.000 + } + + for i := len(ip) - 1; i >= 0; i-- { + octet := ip[i] + + if ipv6 { + // X.X + res.WriteString(strconv.FormatInt(int64(octet&0xf), 16)) + res.WriteRune('.') + res.WriteString(strconv.FormatInt(int64((octet&0xf0)>>4), 16)) + } else { + // X + res.WriteString(strconv.Itoa(int(octet))) + } + + if i != 0 { + res.WriteRune('.') + } + } + return res.String() +} diff --git a/internal/check/dnsbl/common_test.go b/internal/check/dnsbl/common_test.go new file mode 100644 index 0000000..caa0657 --- /dev/null +++ b/internal/check/dnsbl/common_test.go @@ -0,0 +1,238 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dnsbl + +import ( + "context" + "net" + "reflect" + "testing" + + "github.com/foxcpp/go-mockdns" +) + +func TestQueryString(t *testing.T) { + test := func(ip, queryStr string) { + t.Helper() + + parsed := net.ParseIP(ip) + if parsed == nil { + panic("Malformed IP in test") + } + + actual := queryString(parsed) + if actual != queryStr { + t.Errorf("want queryString(%s) to be %s, got %s", ip, queryStr, actual) + } + } + + test("2001:db8:1:2:3:4:567:89ab", "b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.2.0.0.0.1.0.0.0.8.b.d.0.1.0.0.2") + test("2001::1:2:3:4:567:89ab", "b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.2.0.0.0.1.0.0.0.0.0.0.0.1.0.0.2") + test("192.0.2.99", "99.2.0.192") +} + +func TestCheckDomain(t *testing.T) { + test := func(zones map[string]mockdns.Zone, cfg List, domain string, expectedErr error) { + t.Helper() + resolver := mockdns.Resolver{Zones: zones} + err := checkDomain(context.Background(), &resolver, cfg, domain) + if !reflect.DeepEqual(err, expectedErr) { + t.Errorf("expected err to be '%#v', got '%#v'", expectedErr, err) + } + } + + test(nil, List{Zone: "example.org"}, "example.com", nil) + test(map[string]mockdns.Zone{ + "example.com.example.org.": { + Err: &net.DNSError{ + Err: "i/o timeout", + IsTimeout: true, + IsTemporary: true, + }, + }, + }, List{Zone: "example.org"}, "example.com", &net.DNSError{ + Err: "i/o timeout", + IsTimeout: true, + IsTemporary: true, + }) + test(map[string]mockdns.Zone{ + "example.com.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org"}, "example.com", ListedErr{ + Identity: "example.com", + List: "example.org", + Reason: "127.0.0.1", + }) + test(map[string]mockdns.Zone{ + "example.org.example.com.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org"}, "example.com", nil) + test(map[string]mockdns.Zone{ + "example.com.example.org.": { + A: []string{"127.0.0.1"}, + TXT: []string{"Reason"}, + }, + }, List{Zone: "example.org"}, "example.com", ListedErr{ + Identity: "example.com", + List: "example.org", + Reason: "Reason", + }) + test(map[string]mockdns.Zone{ + "example.com.example.org.": { + A: []string{"127.0.0.1"}, + TXT: []string{"Reason 1", "Reason 2"}, + }, + }, List{Zone: "example.org"}, "example.com", ListedErr{ + Identity: "example.com", + List: "example.org", + Reason: "Reason 1; Reason 2", + }) + test(map[string]mockdns.Zone{ + "example.com.example.org.": { + A: []string{"127.0.0.1", "127.0.0.2"}, + }, + }, List{Zone: "example.org"}, "example.com", ListedErr{ + Identity: "example.com", + List: "example.org", + Reason: "127.0.0.1; 127.0.0.2", + }) +} + +func TestCheckIP(t *testing.T) { + test := func(zones map[string]mockdns.Zone, cfg List, ip net.IP, expectedErr error) { + t.Helper() + resolver := mockdns.Resolver{Zones: zones} + err := checkIP(context.Background(), &resolver, cfg, ip) + if !reflect.DeepEqual(err, expectedErr) { + t.Errorf("expected err to be '%#v', got '%#v'", expectedErr, err) + } + } + + test(nil, List{Zone: "example.org"}, net.IPv4(1, 2, 3, 4), nil) + test(nil, List{Zone: "example.org", ClientIPv4: true}, net.IPv4(1, 2, 3, 4), nil) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", ClientIPv4: true}, net.IPv4(1, 2, 3, 4), ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Reason: "127.0.0.1", + }) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"128.0.0.1"}, + }, + }, List{ + Zone: "example.org", + ClientIPv4: true, + Responses: []net.IPNet{ + { + IP: net.IPv4(127, 0, 0, 1), + Mask: net.IPv4Mask(255, 255, 255, 0), + }, + }, + }, net.IPv4(1, 2, 3, 4), nil) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"128.0.0.1"}, + }, + }, List{ + Zone: "example.org", + ClientIPv4: true, + Responses: []net.IPNet{ + { + IP: net.IPv4(127, 0, 0, 0), + Mask: net.IPv4Mask(255, 255, 255, 0), + }, + { + IP: net.IPv4(128, 0, 0, 0), + Mask: net.IPv4Mask(255, 255, 255, 0), + }, + }, + }, net.IPv4(1, 2, 3, 4), ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Reason: "128.0.0.1", + }) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org"}, net.IPv4(1, 2, 3, 4), nil) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + Err: &net.DNSError{ + Err: "i/o timeout", + IsTimeout: true, + IsTemporary: true, + }, + }, + }, List{Zone: "example.org", ClientIPv4: true}, net.IPv4(1, 2, 3, 4), &net.DNSError{ + Err: "i/o timeout", + IsTimeout: true, + IsTemporary: true, + }) + + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + TXT: []string{"Reason"}, + }, + }, List{Zone: "example.org", ClientIPv4: true}, net.IPv4(1, 2, 3, 4), ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Reason: "Reason", + }) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1", "127.0.0.2"}, + }, + }, List{Zone: "example.org", ClientIPv4: true}, net.IPv4(1, 2, 3, 4), ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Reason: "127.0.0.1; 127.0.0.2", + }) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1", "127.0.0.2"}, + TXT: []string{"Reason", "Reason 2"}, + }, + }, List{Zone: "example.org", ClientIPv4: true}, net.IPv4(1, 2, 3, 4), ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Reason: "Reason; Reason 2", + }) + test(map[string]mockdns.Zone{ + "b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.2.0.0.0.1.0.0.0.8.b.d.0.1.0.0.2.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", ClientIPv4: true}, net.ParseIP("2001:db8:1:2:3:4:567:89ab"), nil) + test(map[string]mockdns.Zone{ + "b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.2.0.0.0.1.0.0.0.8.b.d.0.1.0.0.2.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", ClientIPv6: true}, net.ParseIP("2001:db8:1:2:3:4:567:89ab"), ListedErr{ + Identity: "2001:db8:1:2:3:4:567:89ab", + List: "example.org", + Reason: "127.0.0.1", + }) +} diff --git a/internal/check/dnsbl/dnsbl.go b/internal/check/dnsbl/dnsbl.go new file mode 100644 index 0000000..2c91c83 --- /dev/null +++ b/internal/check/dnsbl/dnsbl.go @@ -0,0 +1,435 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dnsbl + +import ( + "context" + "errors" + "net" + "runtime/trace" + "strings" + "sync" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" + "golang.org/x/sync/errgroup" +) + +type List struct { + Zone string + + ClientIPv4 bool + ClientIPv6 bool + + EHLO bool + MAILFROM bool + + ScoreAdj int + Responses []net.IPNet +} + +var defaultBL = List{ + ClientIPv4: true, +} + +type DNSBL struct { + instName string + checkEarly bool + inlineBls []string + bls []List + + quarantineThres int + rejectThres int + + resolver dns.Resolver + log log.Logger +} + +func NewDNSBL(_, instName string, _, inlineArgs []string) (module.Module, error) { + return &DNSBL{ + instName: instName, + inlineBls: inlineArgs, + + resolver: dns.DefaultResolver(), + log: log.Logger{Name: "dnsbl"}, + }, nil +} + +func (bl *DNSBL) Name() string { + return "dnsbl" +} + +func (bl *DNSBL) InstanceName() string { + return bl.instName +} + +func (bl *DNSBL) Init(cfg *config.Map) error { + cfg.Bool("debug", false, false, &bl.log.Debug) + cfg.Bool("check_early", false, false, &bl.checkEarly) + cfg.Int("quarantine_threshold", false, false, 1, &bl.quarantineThres) + cfg.Int("reject_threshold", false, false, 9999, &bl.rejectThres) + cfg.AllowUnknown() + unknown, err := cfg.Process() + if err != nil { + return err + } + + for _, inlineBl := range bl.inlineBls { + cfg := defaultBL + cfg.Zone = inlineBl + go bl.testList(cfg) + bl.bls = append(bl.bls, cfg) + } + + for _, node := range unknown { + if err := bl.readListCfg(node); err != nil { + return err + } + } + + return nil +} + +func (bl *DNSBL) readListCfg(node config.Node) error { + var ( + listCfg List + responseNets []string + ) + + cfg := config.NewMap(nil, node) + cfg.Bool("client_ipv4", false, defaultBL.ClientIPv4, &listCfg.ClientIPv4) + cfg.Bool("client_ipv6", false, defaultBL.ClientIPv4, &listCfg.ClientIPv6) + cfg.Bool("ehlo", false, defaultBL.EHLO, &listCfg.EHLO) + cfg.Bool("mailfrom", false, defaultBL.EHLO, &listCfg.MAILFROM) + cfg.Int("score", false, false, 1, &listCfg.ScoreAdj) + cfg.StringList("responses", false, false, []string{"127.0.0.1/24"}, &responseNets) + if _, err := cfg.Process(); err != nil { + return err + } + + for _, resp := range responseNets { + // If there is no / - it is a plain IP address, append + // '/32'. + if !strings.Contains(resp, "/") { + resp += "/32" + } + + _, ipNet, err := net.ParseCIDR(resp) + if err != nil { + return err + } + listCfg.Responses = append(listCfg.Responses, *ipNet) + } + + for _, zone := range append([]string{node.Name}, node.Args...) { + zoneCfg := listCfg + zoneCfg.Zone = zone + + if listCfg.ScoreAdj < 0 { + if zoneCfg.EHLO { + return errors.New("dnsbl: 'ehlo' should not be used with negative score") + } + if zoneCfg.MAILFROM { + return errors.New("dnsbl: 'mailfrom' should not be used with negative score") + } + } + bl.bls = append(bl.bls, zoneCfg) + + // From RFC 5782 Section 7: + // >To avoid this situation, systems that use + // >DNSxLs SHOULD check for the test entries described in Section 5 to + // >ensure that a domain actually has the structure of a DNSxL, and + // >SHOULD NOT use any DNSxL domain that does not have correct test + // >entries. + // Sadly, however, many DNSBLs lack test records so at most we can + // log a warning. Also, DNS is kinda slow so we do checks + // asynchronously to prevent slowing down server start-up. + go bl.testList(zoneCfg) + } + + return nil +} + +func (bl *DNSBL) testList(listCfg List) { + // Check RFC 5782 Section 5 requirements. + + bl.log.DebugMsg("testing list for RFC 5782 requirements...", "list", listCfg.Zone) + + // 1. IPv4-based DNSxLs MUST contain an entry for 127.0.0.2 for testing purposes. + if listCfg.ClientIPv4 { + err := checkIP(context.Background(), bl.resolver, listCfg, net.IPv4(127, 0, 0, 2)) + if err == nil { + bl.log.Msg("List does not contain a test record for 127.0.0.2", "list", listCfg.Zone) + } else if _, ok := err.(ListedErr); !ok { + bl.log.Error("lookup error, bailing out", err, "list", listCfg.Zone) + return + } + + // 2. IPv4-based DNSxLs MUST NOT contain an entry for 127.0.0.1. + err = checkIP(context.Background(), bl.resolver, listCfg, net.IPv4(127, 0, 0, 1)) + if err != nil { + _, ok := err.(ListedErr) + if !ok { + bl.log.Error("lookup error, bailing out", err, "list", listCfg.Zone) + return + } + bl.log.Msg("List contains a record for 127.0.0.1", "list", listCfg.Zone) + } + } + + if listCfg.ClientIPv6 { + // 1. IPv6-based DNSxLs MUST contain an entry for ::FFFF:7F00:2 + mustIP := net.ParseIP("::FFFF:7F00:2") + + err := checkIP(context.Background(), bl.resolver, listCfg, mustIP) + if err == nil { + bl.log.Msg("List does not contain a test record for ::FFFF:7F00:2", "list", listCfg.Zone) + } else if _, ok := err.(ListedErr); !ok { + bl.log.Error("lookup error, bailing out", err, "list", listCfg.Zone) + return + } + + // 2. IPv4-based DNSxLs MUST NOT contain an entry for ::FFFF:7F00:1 + mustNotIP := net.ParseIP("::FFFF:7F00:1") + err = checkIP(context.Background(), bl.resolver, listCfg, mustNotIP) + if err != nil { + _, ok := err.(ListedErr) + if !ok { + bl.log.Error("lookup error, bailing out", err, "list", listCfg.Zone) + return + } + bl.log.Msg("List contains a record for ::FFFF:7F00:1", "list", listCfg.Zone) + } + } + + if listCfg.EHLO || listCfg.MAILFROM { + // Domain-name-based DNSxLs MUST contain an entry for the reserved + // domain name "TEST". + err := checkDomain(context.Background(), bl.resolver, listCfg, "test") + if err == nil { + bl.log.Msg("List does not contain a test record for 'test' TLD", "list", listCfg.Zone) + } else if _, ok := err.(ListedErr); !ok { + bl.log.Error("lookup error, bailing out", err, "list", listCfg.Zone) + return + } + + // ... and MUST NOT contain an entry for the reserved domain name + // "INVALID". + err = checkDomain(context.Background(), bl.resolver, listCfg, "invalid") + if err != nil { + _, ok := err.(ListedErr) + if !ok { + bl.log.Error("lookup error, bailing out", err, "list", listCfg.Zone) + return + } + bl.log.Msg("List contains a record for 'invalid' TLD", "list", listCfg.Zone) + } + } +} + +func (bl *DNSBL) checkList(ctx context.Context, list List, ip net.IP, ehlo, mailFrom string) error { + if list.ClientIPv4 || list.ClientIPv6 { + if err := checkIP(ctx, bl.resolver, list, ip); err != nil { + return err + } + } + + if list.EHLO && ehlo != "" { + // Skip IPs in EHLO. + if strings.HasPrefix(ehlo, "[") && strings.HasSuffix(ehlo, "]") { + return nil + } + + if err := checkDomain(ctx, bl.resolver, list, ehlo); err != nil { + return err + } + } + + if list.MAILFROM && mailFrom != "" { + _, domain, err := address.Split(mailFrom) + if err != nil || domain == "" { + // Probably or <>, not much we can check. + return nil + } + + // If EHLO == domain (usually the case for small/private email servers) + // then don't do a second lookup for the same domain. + if list.EHLO && dns.Equal(domain, ehlo) { + return nil + } + + if err := checkDomain(ctx, bl.resolver, list, domain); err != nil { + return err + } + } + + return nil +} + +func (bl *DNSBL) checkLists(ctx context.Context, ip net.IP, ehlo, mailFrom string) module.CheckResult { + var ( + eg = errgroup.Group{} + + // Protects variables below. + lck sync.Mutex + score int + listedOn []string + reasons []string + ) + + for _, list := range bl.bls { + eg.Go(func() error { + err := bl.checkList(ctx, list, ip, ehlo, mailFrom) + if err != nil { + listErr, listed := err.(ListedErr) + if !listed { + return err + } + + lck.Lock() + defer lck.Unlock() + listedOn = append(listedOn, listErr.List) + reasons = append(reasons, listErr.Reason) + score += list.ScoreAdj + } + return nil + }) + } + + err := eg.Wait() + if err != nil { + // Lookup error for BL, hard-fail. + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: exterrors.SMTPCode(err, 451, 554), + EnhancedCode: exterrors.SMTPEnchCode(err, exterrors.EnhancedCode{0, 7, 0}), + Message: "DNS error during policy check", + Err: err, + CheckName: "dnsbl", + }, + } + } + + if score >= bl.rejectThres { + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Client identity is listed in the used DNSBL", + Err: err, + CheckName: "dnsbl", + }, + } + } + if score >= bl.quarantineThres { + return module.CheckResult{ + Quarantine: true, + Reason: &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Client identity is listed in the used DNSBL", + Err: err, + CheckName: "dnsbl", + }, + } + } + + return module.CheckResult{} +} + +// CheckConnection implements module.EarlyCheck. +func (bl *DNSBL) CheckConnection(ctx context.Context, state *module.ConnState) error { + defer trace.StartRegion(ctx, "dnsbl/CheckConnection (Early)").End() + + ip, ok := state.RemoteAddr.(*net.TCPAddr) + if !ok { + bl.log.Msg("non-TCP/IP source", + "src_addr", state.RemoteAddr, + "src_host", state.Hostname) + return nil + } + + result := bl.checkLists(ctx, ip.IP, state.Hostname, "") + if result.Reject && bl.checkEarly { + return result.Reason + } + + state.ModData.Set(bl, true, result) + + return nil +} + +type state struct { + bl *DNSBL + msgMeta *module.MsgMetadata + log log.Logger +} + +func (bl *DNSBL) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &state{ + bl: bl, + msgMeta: msgMeta, + log: target.DeliveryLogger(bl.log, msgMeta), + }, nil +} + +func (s *state) CheckConnection(ctx context.Context) module.CheckResult { + defer trace.StartRegion(ctx, "dnsbl/CheckConnection").End() + + if s.msgMeta.Conn == nil { + s.log.Msg("locally generated message, ignoring") + return module.CheckResult{} + } + + result := s.msgMeta.Conn.ModData.Get(s.bl, true) + if result != nil { + return result.(module.CheckResult) + } + + return module.CheckResult{} +} + +func (*state) CheckSender(context.Context, string) module.CheckResult { + return module.CheckResult{} +} + +func (*state) CheckRcpt(context.Context, string) module.CheckResult { + return module.CheckResult{} +} + +func (*state) CheckBody(context.Context, textproto.Header, buffer.Buffer) module.CheckResult { + return module.CheckResult{} +} + +func (*state) Close() error { + return nil +} + +func init() { + module.Register("check.dnsbl", NewDNSBL) +} diff --git a/internal/check/dnsbl/dnsbl_test.go b/internal/check/dnsbl/dnsbl_test.go new file mode 100644 index 0000000..1845aeb --- /dev/null +++ b/internal/check/dnsbl/dnsbl_test.go @@ -0,0 +1,213 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dnsbl + +import ( + "context" + "errors" + "net" + "testing" + + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestCheckList(t *testing.T) { + test := func(zones map[string]mockdns.Zone, cfg List, ip net.IP, ehlo, mailFrom string, expectedErr error) { + mod := &DNSBL{ + resolver: &mockdns.Resolver{Zones: zones}, + log: testutils.Logger(t, "dnsbl"), + } + err := mod.checkList(context.Background(), cfg, ip, ehlo, mailFrom) + if !errors.Is(err, expectedErr) { + t.Errorf("expected err to be '%#v', got '%#v'", expectedErr, err) + } + } + + test(nil, List{Zone: "example.org"}, net.IPv4(1, 2, 3, 4), + "example.com", "foo@example.com", nil) + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", ClientIPv4: true}, net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", ListedErr{ + Identity: "1.2.3.4", + List: "example.org", + Reason: "127.0.0.1", + }, + ) + test(map[string]mockdns.Zone{ + "mx.example.com.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org"}, net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", nil, + ) + test(map[string]mockdns.Zone{ + "mx.example.com.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", EHLO: true}, net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", ListedErr{ + Identity: "mx.example.com", + List: "example.org", + Reason: "127.0.0.1", + }, + ) + test(map[string]mockdns.Zone{ + "[1.2.3.4].example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", EHLO: true}, net.IPv4(1, 2, 3, 4), + "[1.2.3.4]", "foo@example.com", nil, + ) + test(map[string]mockdns.Zone{ + "[IPv6:beef::1].example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", EHLO: true}, net.IPv4(1, 2, 3, 4), + "[IPv6:beef::1]", "foo@example.com", nil, + ) + test(map[string]mockdns.Zone{ + "example.com.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org"}, net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", nil, + ) + test(map[string]mockdns.Zone{ + "postmaster.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", MAILFROM: true}, net.IPv4(1, 2, 3, 4), + "mx.example.com", "postmaster", nil, + ) + test(map[string]mockdns.Zone{ + ".example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", MAILFROM: true}, net.IPv4(1, 2, 3, 4), + "mx.example.com", "", nil, + ) + test(map[string]mockdns.Zone{ + "example.com.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, List{Zone: "example.org", MAILFROM: true}, net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", ListedErr{ + Identity: "example.com", + List: "example.org", + Reason: "127.0.0.1", + }, + ) +} + +func TestCheckLists(t *testing.T) { + test := func(zones map[string]mockdns.Zone, bls []List, ip net.IP, ehlo, mailFrom string, reject, quarantine bool) { + mod := &DNSBL{ + bls: bls, + resolver: &mockdns.Resolver{Zones: zones}, + log: testutils.Logger(t, "dnsbl"), + quarantineThres: 1, + rejectThres: 2, + } + result := mod.checkLists(context.Background(), ip, ehlo, mailFrom) + + if result.Reject && !reject { + t.Errorf("Expected message to not be rejected") + } + if !result.Reject && reject { + t.Errorf("Expected message to be rejected") + } + if result.Quarantine && !quarantine { + t.Errorf("Expected message to not be quarantined") + } + if !result.Quarantine && quarantine { + t.Errorf("Expected message to be quarantined") + } + } + + // Score 2 >= 2, reject + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, []List{ + { + Zone: "example.org", + ClientIPv4: true, + ScoreAdj: 2, + }, + }, net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", true, false, + ) + + // Score 1 >= 1, quarantine + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + }, + }, []List{ + { + Zone: "example.org", + ClientIPv4: true, + ScoreAdj: 1, + }, + }, net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", false, true, + ) + + // Score 0, no action + test(map[string]mockdns.Zone{ + "4.3.2.1.example.org.": { + A: []string{"127.0.0.1"}, + }, + "4.3.2.1.example.net.": { + A: []string{"127.0.0.1"}, + }, + }, + []List{ + {Zone: "example.org", ClientIPv4: true, ScoreAdj: 1}, + {Zone: "example.net", ClientIPv4: true, ScoreAdj: -1}, + }, + net.IPv4(1, 2, 3, 4), + "mx.example.com", "foo@example.com", + false, false, + ) + + // DNS error, hard-fail (reject) + test(map[string]mockdns.Zone{ + "4.3.2.2.example.org.": { + Err: &net.DNSError{ + Err: "i/o timeout", + IsTimeout: true, + IsTemporary: true, + }, + }, + }, + []List{ + {Zone: "example.org", ClientIPv4: true, ScoreAdj: 1}, + {Zone: "example.net", ClientIPv4: true, ScoreAdj: 2}, + }, + net.IPv4(2, 2, 3, 4), + "mx.example.com", "foo@example.com", + true, false, + ) +} diff --git a/internal/check/milter/milter.go b/internal/check/milter/milter.go new file mode 100644 index 0000000..37704d4 --- /dev/null +++ b/internal/check/milter/milter.go @@ -0,0 +1,446 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package milter + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-milter" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +const modName = "check.milter" + +type Check struct { + cl *milter.Client + milterUrl string + failOpen bool + instName string + log log.Logger +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + c := &Check{ + instName: instName, + log: log.Logger{Name: modName, Debug: log.DefaultLogger.Debug}, + } + switch len(inlineArgs) { + case 1: + c.milterUrl = inlineArgs[0] + case 0: + default: + return nil, fmt.Errorf("%s: unexpected amount of arguments, want 1 or 0", modName) + } + return c, nil +} + +func (c *Check) Name() string { + return modName +} + +func (c *Check) InstanceName() string { + return c.instName +} + +func (c *Check) Init(cfg *config.Map) error { + cfg.String("endpoint", false, false, c.milterUrl, &c.milterUrl) + cfg.Bool("fail_open", false, false, &c.failOpen) + if _, err := cfg.Process(); err != nil { + return err + } + + if c.milterUrl == "" { + return fmt.Errorf("%s: milter endpoint is not set", modName) + } + + endp, err := config.ParseEndpoint(c.milterUrl) + if err != nil { + return fmt.Errorf("%s: %v", modName, err) + } + + switch endp.Scheme { + case "tcp", "unix": + default: + return fmt.Errorf("%s: scheme unsupported: %v", modName, endp.Scheme) + } + + c.cl = milter.NewClientWithOptions(endp.Network(), endp.Address(), milter.ClientOptions{ + Dialer: &net.Dialer{ + Timeout: 10 * time.Second, + }, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + ActionMask: milter.OptAddHeader | milter.OptQuarantine, + ProtocolMask: 0, + }) + + return nil +} + +type state struct { + c *Check + session *milter.ClientSession + msgMeta *module.MsgMetadata + skipChecks bool + log log.Logger +} + +func (c *Check) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + session, err := c.cl.Session() + if err != nil { + return nil, err + } + return &state{ + c: c, + session: session, + msgMeta: msgMeta, + log: target.DeliveryLogger(c.log, msgMeta), + }, nil +} + +func (s *state) handleAction(act *milter.Action) module.CheckResult { + switch act.Code { + case milter.ActAccept: + s.skipChecks = true + return module.CheckResult{} + case milter.ActContinue: + return module.CheckResult{} + case milter.ActReplyCode: + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: act.SMTPCode, + EnhancedCode: exterrors.EnhancedCode{5, 7, 1}, + Message: "Message rejected due to local policy", + Reason: "reply code action", + CheckName: "milter", + Misc: map[string]interface{}{ + "milter": s.c.milterUrl, + }, + }, + } + case milter.ActDiscard: + s.log.Msg("silent discard is not supported, rejecting message") + fallthrough + case milter.ActTempFail: + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 450, + EnhancedCode: exterrors.EnhancedCode{4, 7, 1}, + Message: "Message rejected due to local policy", + Reason: "reject action", + CheckName: "milter", + Misc: map[string]interface{}{ + "milter": s.c.milterUrl, + }, + }, + } + case milter.ActReject: + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 1}, + Message: "Message rejected due to local policy", + Reason: "reject action", + CheckName: "milter", + Misc: map[string]interface{}{ + "milter": s.c.milterUrl, + }, + }, + } + default: + s.log.Msg("unknown action code ignored", "code", act.Code, "milter", s.c.milterUrl) + return module.CheckResult{} + } +} + +// apply applies the modification actions returned by milter to the check results object. +func (s *state) apply(modifyActs []milter.ModifyAction, res module.CheckResult) module.CheckResult { + out := res + for _, act := range modifyActs { + switch act.Code { + case milter.ActAddRcpt, milter.ActDelRcpt: + s.log.Msg("envelope changes are not supported", "rcpt", act.Rcpt, "code", act.Code, "milter", s.c.milterUrl) + case milter.ActChangeFrom: + s.log.Msg("envelope changes are not supported", "from", act.From, "code", act.Code, "milter", s.c.milterUrl) + case milter.ActChangeHeader: + s.log.Msg("header field changes are not supported", "field", act.HeaderName, "milter", s.c.milterUrl) + case milter.ActInsertHeader: + if act.HeaderIndex != 1 { + s.log.Msg("header inserting not on top is not supported, prepending instead", "field", act.HeaderName, "milter", s.c.milterUrl) + } + fallthrough + case milter.ActAddHeader: + // Header field might be arbitarly folded by the caller and we want + // to preserve that exact format in case it is important (DKIM + // signature is added by milter). + field := make([]byte, 0, len(act.HeaderName)+2+len(act.HeaderValue)+2) + field = append(field, act.HeaderName...) + field = append(field, ':', ' ') + field = append(field, act.HeaderValue...) + field = append(field, '\r', '\n') + out.Header.AddRaw(field) + case milter.ActQuarantine: + out.Quarantine = true + out.Reason = exterrors.WithFields(errors.New("milter quarantine action"), map[string]interface{}{ + "check": "milter", + "milter": s.c.milterUrl, + "reason": act.Reason, + }) + } + } + return out +} + +func (s *state) CheckConnection(ctx context.Context) module.CheckResult { + if s.msgMeta.Conn == nil { + // Submit some dummy values as the message is likely generated locally. + + act, err := s.session.Conn("localhost", milter.FamilyInet, 25, "127.0.0.1") + if err != nil { + return s.ioError(err) + } + if act.Code != milter.ActContinue { + return s.handleAction(act) + } + + act, err = s.session.Helo("localhost") + if err != nil { + return s.ioError(err) + } + return s.handleAction(act) + } + + if !s.session.ProtocolOption(milter.OptNoConnect) { + if err := s.session.Macros(milter.CodeConn, + "daemon_name", "maddy", + "if_name", "unknown", + "if_addr", "0.0.0.0", + // TODO: $j + // TODO: $_ + ); err != nil { + return s.ioError(err) + } + + var ( + protoFamily milter.ProtoFamily + port uint16 + addr string + ) + switch rAddr := s.msgMeta.Conn.RemoteAddr.(type) { + case *net.TCPAddr: + port = uint16(rAddr.Port) + if v4 := rAddr.IP.To4(); v4 != nil { + // Make sure to not accidentally send IPv6-mapped IPv4 address. + protoFamily = milter.FamilyInet + addr = v4.String() + } else { + protoFamily = milter.FamilyInet6 + addr = rAddr.IP.String() + } + case *net.UnixAddr: + protoFamily = milter.FamilyUnix + addr = rAddr.Name + default: + protoFamily = milter.FamilyUnknown + } + + act, err := s.session.Conn(s.msgMeta.Conn.Hostname, protoFamily, port, addr) + if err != nil { + return s.ioError(err) + } + if act.Code != milter.ActContinue { + return s.handleAction(act) + } + } + + if !s.session.ProtocolOption(milter.OptNoHelo) { + if s.msgMeta.Conn.TLS.HandshakeComplete { + fields := make([]string, 0, 4*2) + tlsState := s.msgMeta.Conn.TLS + + switch tlsState.Version { + case tls.VersionTLS10: + fields = append(fields, "tls_version", "TLSv1") + case tls.VersionTLS11: + fields = append(fields, "tls_version", "TLSv1.1") + case tls.VersionTLS12: + fields = append(fields, "tls_version", "TLSv1.2") + case tls.VersionTLS13: + fields = append(fields, "tls_version", "TLSv1.3") + } + fields = append(fields, "cipher", tls.CipherSuiteName(tlsState.CipherSuite)) + + if len(tlsState.PeerCertificates) != 0 { + fields = append(fields, "cert_subject", + tlsState.PeerCertificates[len(tlsState.PeerCertificates)-1].Subject.String()) + fields = append(fields, "cert_issuer", + tlsState.PeerCertificates[len(tlsState.PeerCertificates)-1].Issuer.String()) + } + + if err := s.session.Macros(milter.CodeHelo, fields...); err != nil { + return s.ioError(err) + } + } + act, err := s.session.Helo(s.msgMeta.Conn.Hostname) + if err != nil { + return s.ioError(err) + } + return s.handleAction(act) + } + + return module.CheckResult{} +} + +func (s *state) ioError(err error) module.CheckResult { + if s.c.failOpen { + s.skipChecks = true // silently permit processing to continue + s.c.log.Error("I/O error", err) + return module.CheckResult{} + } + + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 1}, + Message: "I/O error during policy check", + Err: err, + CheckName: "milter", + Misc: map[string]interface{}{ + "milter": s.c.milterUrl, + }, + }, + } +} + +func (s *state) CheckSender(ctx context.Context, mailFrom string) module.CheckResult { + if s.skipChecks || s.session.ProtocolOption(milter.OptNoMailFrom) { + return module.CheckResult{} + } + + fields := make([]string, 0, 2) + fields = append(fields, "i", s.msgMeta.ID) + // TODO: fields = append(fields, "auth_type", s.msgMeta.???) + if s.msgMeta.Conn.AuthUser != "" { + fields = append(fields, "auth_authen", s.msgMeta.Conn.AuthUser) + } + if err := s.session.Macros(milter.CodeMail, fields...); err != nil { + return s.ioError(err) + } + + esmtpArgs := make([]string, 0, 2) + if s.msgMeta.SMTPOpts.UTF8 { + esmtpArgs = append(esmtpArgs, "SMTPUTF8") + } + + act, err := s.session.Mail(mailFrom, esmtpArgs) + if err != nil { + return s.ioError(err) + } + return s.handleAction(act) +} + +func (s *state) CheckRcpt(ctx context.Context, rcptTo string) module.CheckResult { + if s.skipChecks { + return module.CheckResult{} + } + + act, err := s.session.Rcpt(rcptTo, nil) + if err != nil { + return s.ioError(err) + } + return s.handleAction(act) +} + +func (s *state) CheckBody(ctx context.Context, header textproto.Header, body buffer.Buffer) module.CheckResult { + if s.skipChecks { + return module.CheckResult{} + } + + act, err := s.session.Header(header) + if err != nil { + return s.ioError(err) + } + if act.Code != milter.ActContinue { + return s.handleAction(act) + } + + var modifyAct []milter.ModifyAction + + if !s.session.ProtocolOption(milter.OptNoBody) { + // body.Open can be expensive for on-disk buffering. + r, err := body.Open() + if err != nil { + // Not ioError(err) because fail_open directive is applied only for external I/O. + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 1}, + Message: "Internal error during policy check", + Err: err, + CheckName: "milter", + Misc: map[string]interface{}{ + "milter": s.c.milterUrl, + }, + }, + } + } + + modifyAct, act, err = s.session.BodyReadFrom(r) + if err != nil { + return s.ioError(err) + } + } else { + modifyAct, act, err = s.session.End() + if err != nil { + return s.ioError(err) + } + } + + result := s.handleAction(act) + return s.apply(modifyAct, result) +} + +func (s *state) Close() error { + return s.session.Close() +} + +var ( + _ module.Check = &Check{} + _ module.CheckState = &state{} +) + +func init() { + module.Register(modName, New) +} diff --git a/internal/check/milter/milter_test.go b/internal/check/milter/milter_test.go new file mode 100644 index 0000000..9797851 --- /dev/null +++ b/internal/check/milter/milter_test.go @@ -0,0 +1,61 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package milter + +import ( + "testing" + + "github.com/foxcpp/maddy/framework/config" +) + +func TestAcceptValidEndpoints(t *testing.T) { + for _, endpoint := range []string{ + "tcp://0.0.0.0:10025", + "tcp://[::]:10025", + "tcp:127.0.0.1:10025", + "unix://path", + "unix:path", + "unix:/path", + "unix:///path", + "unix://also/path", + "unix:///also/path", + } { + c := &Check{milterUrl: endpoint} + + err := c.Init(&config.Map{}) + if err != nil { + t.Errorf("Unexpected failure for %s: %v", endpoint, err) + return + } + } +} + +func TestRejectInvalidEndpoints(t *testing.T) { + for _, endpoint := range []string{ + "tls://0.0.0.0:10025", + "tls:0.0.0.0:10025", + } { + c := &Check{milterUrl: endpoint} + err := c.Init(&config.Map{}) + if err == nil { + t.Errorf("Accepted invalid endpoint: %s", endpoint) + return + } + } +} diff --git a/internal/check/requiretls/requiretls.go b/internal/check/requiretls/requiretls.go new file mode 100644 index 0000000..bdd2f26 --- /dev/null +++ b/internal/check/requiretls/requiretls.go @@ -0,0 +1,45 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package requiretls + +import ( + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/check" +) + +func requireTLS(ctx check.StatelessCheckContext) module.CheckResult { + if ctx.MsgMeta.Conn != nil && ctx.MsgMeta.Conn.TLS.HandshakeComplete { + return module.CheckResult{} + } + + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 1}, + Message: "TLS conversation required", + CheckName: "require_tls", + }, + } +} + +func init() { + check.RegisterStatelessCheck("require_tls", modconfig.FailAction{Reject: true}, requireTLS, nil, nil, nil) +} diff --git a/internal/check/rspamd/rspamd.go b/internal/check/rspamd/rspamd.go new file mode 100644 index 0000000..e6afad6 --- /dev/null +++ b/internal/check/rspamd/rspamd.go @@ -0,0 +1,367 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package rspamd + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strconv" + "strings" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + tls2 "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +const modName = "check.rspamd" + +type Check struct { + instName string + log log.Logger + + apiPath string + flags string + settingsID string + tag string + mtaName string + + ioErrAction modconfig.FailAction + errorRespAction modconfig.FailAction + addHdrAction modconfig.FailAction + rewriteSubjAction modconfig.FailAction + + client *http.Client +} + +func New(modName, instName string, _, inlineArgs []string) (module.Module, error) { + c := &Check{ + instName: instName, + client: http.DefaultClient, + log: log.Logger{Name: modName, Debug: log.DefaultLogger.Debug}, + } + + switch len(inlineArgs) { + case 1: + c.apiPath = inlineArgs[0] + case 0: + c.apiPath = "http://127.0.0.1:11333" + default: + return nil, fmt.Errorf("%s: unexpected amount of inline arguments", modName) + } + + return c, nil +} + +func (c *Check) Name() string { + return modName +} + +func (c *Check) InstanceName() string { + return c.instName +} + +func (c *Check) Init(cfg *config.Map) error { + var ( + tlsConfig tls.Config + flags []string + ) + + cfg.Custom("tls_client", true, false, func() (interface{}, error) { + return tls.Config{}, nil + }, tls2.TLSClientBlock, &tlsConfig) + cfg.String("api_path", false, false, c.apiPath, &c.apiPath) + cfg.String("settings_id", false, false, "", &c.settingsID) + cfg.String("tag", false, false, "maddy", &c.tag) + cfg.String("hostname", true, false, "", &c.mtaName) + cfg.Custom("io_error_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.ioErrAction) + cfg.Custom("error_resp_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.errorRespAction) + cfg.Custom("add_header_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{Quarantine: true}, nil + }, modconfig.FailActionDirective, &c.addHdrAction) + cfg.Custom("rewrite_subj_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{Quarantine: true}, nil + }, modconfig.FailActionDirective, &c.rewriteSubjAction) + cfg.StringList("flags", false, false, []string{"pass_all"}, &flags) + if _, err := cfg.Process(); err != nil { + return err + } + + c.client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tlsConfig, + }, + } + c.flags = strings.Join(flags, ",") + + return nil +} + +type state struct { + c *Check + msgMeta *module.MsgMetadata + log log.Logger + + mailFrom string + rcpt []string +} + +func (c *Check) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &state{ + c: c, + msgMeta: msgMeta, + log: target.DeliveryLogger(c.log, msgMeta), + }, nil +} + +func (s *state) CheckConnection(ctx context.Context) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckSender(ctx context.Context, addr string) module.CheckResult { + s.mailFrom = addr + return module.CheckResult{} +} + +func (s *state) CheckRcpt(ctx context.Context, addr string) module.CheckResult { + s.rcpt = append(s.rcpt, addr) + return module.CheckResult{} +} + +func addConnHeaders(r *http.Request, meta *module.MsgMetadata, mailFrom string, rcpts []string) { + r.Header.Add("From", mailFrom) + for _, rcpt := range rcpts { + r.Header.Add("Rcpt", rcpt) + } + + r.Header.Add("Queue-ID", meta.ID) + + conn := meta.Conn + if conn != nil { + if meta.Conn.AuthUser != "" { + r.Header.Add("User", meta.Conn.AuthUser) + } + + if tcpAddr, ok := conn.RemoteAddr.(*net.TCPAddr); ok { + r.Header.Add("IP", tcpAddr.IP.String()) + } + r.Header.Add("Helo", conn.Hostname) + name, err := conn.RDNSName.Get() + if err == nil && name != nil { + r.Header.Add("Hostname", name.(string)) + } + + if conn.TLS.HandshakeComplete { + r.Header.Add("TLS-Cipher", tls.CipherSuiteName(conn.TLS.CipherSuite)) + switch conn.TLS.Version { + case tls.VersionTLS13: + r.Header.Add("TLS-Version", "1.3") + case tls.VersionTLS12: + r.Header.Add("TLS-Version", "1.2") + case tls.VersionTLS11: + r.Header.Add("TLS-Version", "1.1") + case tls.VersionTLS10: + r.Header.Add("TLS-Version", "1.0") + } + } + } +} + +func (s *state) CheckBody(ctx context.Context, hdr textproto.Header, body buffer.Buffer) module.CheckResult { + bodyR, err := body.Open() + if err != nil { + return module.CheckResult{ + Reject: true, + Reason: exterrors.WithFields(err, map[string]interface{}{"check": modName}), + } + } + + var buf bytes.Buffer + if err := textproto.WriteHeader(&buf, hdr); err != nil { + return module.CheckResult{ + Reject: true, + Reason: exterrors.WithFields(err, map[string]interface{}{"check": modName}), + } + } + + r, err := http.NewRequest("POST", s.c.apiPath+"/checkv2", io.MultiReader(&buf, bodyR)) + if err != nil { + return module.CheckResult{ + Reject: true, + Reason: exterrors.WithFields(err, map[string]interface{}{"check": modName}), + } + } + + r.Header.Add("Pass", "all") // TODO: does that need to be configurable? + // TODO: include version (needs maddy.Version moved somewhere to break circular dependency) + r.Header.Add("User-Agent", "maddy") + if s.c.tag != "" { + r.Header.Add("MTA-Tag", s.c.tag) + } + if s.c.settingsID != "" { + r.Header.Add("Settings-ID", s.c.settingsID) + } + if s.c.mtaName != "" { + r.Header.Add("MTA-Name", s.c.mtaName) + } + + addConnHeaders(r, s.msgMeta, s.mailFrom, s.rcpt) + r.Header.Add("Content-Length", strconv.Itoa(body.Len())) + + resp, err := s.c.client.Do(r) + if err != nil { + return s.c.ioErrAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Internal error during policy check", + CheckName: modName, + Err: err, + }, + }) + } + if resp.StatusCode/100 != 2 { + return s.c.errorRespAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Internal error during policy check", + CheckName: modName, + Err: fmt.Errorf("HTTP %d", resp.StatusCode), + }, + }) + } + defer resp.Body.Close() + + var respData response + if err := json.NewDecoder(resp.Body).Decode(&respData); err != nil { + return s.c.ioErrAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 9, 0}, + Message: "Internal error during policy check", + CheckName: modName, + Err: err, + }, + }) + } + + switch respData.Action { + case "no action": + return module.CheckResult{} + case "greylist": + // uuh... TODO: Implement greylisting? + hdrAdd := textproto.Header{} + hdrAdd.Add("X-Spam-Score", strconv.FormatFloat(respData.Score, 'f', 2, 64)) + return module.CheckResult{ + Header: hdrAdd, + } + case "add header": + hdrAdd := textproto.Header{} + hdrAdd.Add("X-Spam-Flag", "Yes") + hdrAdd.Add("X-Spam-Score", strconv.FormatFloat(respData.Score, 'f', 2, 64)) + return s.c.addHdrAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 450, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Message rejected due to local policy", + CheckName: modName, + Misc: map[string]interface{}{"action": "add header"}, + }, + Header: hdrAdd, + }) + case "rewrite subject": + hdrAdd := textproto.Header{} + hdrAdd.Add("X-Spam-Flag", "Yes") + hdrAdd.Add("X-Spam-Score", strconv.FormatFloat(respData.Score, 'f', 2, 64)) + return s.c.rewriteSubjAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 450, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Message rejected due to local policy", + CheckName: modName, + Misc: map[string]interface{}{"action": "rewrite subject"}, + }, + Header: hdrAdd, + }) + case "soft reject": + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 450, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Message rejected due to local policy", + CheckName: modName, + Misc: map[string]interface{}{"action": "soft reject"}, + }, + } + case "reject": + return module.CheckResult{ + Reject: true, + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Message rejected due to local policy", + CheckName: modName, + Misc: map[string]interface{}{"action": "reject"}, + }, + } + } + + s.log.Msg("unhandled action", "action", respData.Action) + + return module.CheckResult{} +} + +type response struct { + Score float64 `json:"score"` + Action string `json:"action"` + Subject string `json:"subject"` + Symbols map[string]struct { + Name string `json:"name"` + Score float64 `json:"score"` + } +} + +func (s *state) Close() error { + return nil +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/check/skeleton.go b/internal/check/skeleton.go new file mode 100644 index 0000000..f374359 --- /dev/null +++ b/internal/check/skeleton.go @@ -0,0 +1,101 @@ +//go:build ignore +// +build ignore + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +/* +This is example of a minimal stateful check module implementation. +See HACKING.md in the repo root for implementation recommendations. +*/ + +package directory_name_here + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +const modName = "check_things" + +type Check struct { + instName string + log log.Logger +} + +func New(modName, instName string, aliases, inlineArgs []string) (module.Module, error) { + return &Check{ + instName: instName, + }, nil +} + +func (c *Check) Name() string { + return modName +} + +func (c *Check) InstanceName() string { + return c.instName +} + +func (c *Check) Init(cfg *config.Map) error { + return nil +} + +type state struct { + c *Check + msgMeta *module.MsgMetadata + log log.Logger +} + +func (c *Check) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &state{ + c: c, + msgMeta: msgMeta, + log: target.DeliveryLogger(c.log, msgMeta), + }, nil +} + +func (s *state) CheckConnection(ctx context.Context) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckSender(ctx context.Context, addr string) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckRcpt(ctx context.Context, addr string) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckBody(ctx context.Context, hdr textproto.Header, body buffer.Buffer) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) Close() error { + return nil +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/check/spf/spf.go b/internal/check/spf/spf.go new file mode 100644 index 0000000..c94dd91 --- /dev/null +++ b/internal/check/spf/spf.go @@ -0,0 +1,420 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package spf + +import ( + "context" + "errors" + "fmt" + "net" + "runtime/debug" + "runtime/trace" + + "blitiri.com.ar/go/spf" + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/emersion/go-msgauth/dmarc" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + maddydmarc "github.com/foxcpp/maddy/internal/dmarc" + "github.com/foxcpp/maddy/internal/target" + "golang.org/x/net/idna" +) + +const modName = "check.spf" + +type Check struct { + instName string + enforceEarly bool + + noneAction modconfig.FailAction + neutralAction modconfig.FailAction + failAction modconfig.FailAction + softfailAction modconfig.FailAction + permerrAction modconfig.FailAction + temperrAction modconfig.FailAction + + log log.Logger + resolver dns.Resolver +} + +func New(_, instName string, _, _ []string) (module.Module, error) { + return &Check{ + instName: instName, + log: log.Logger{Name: modName}, + resolver: dns.DefaultResolver(), + }, nil +} + +func (c *Check) Name() string { + return modName +} + +func (c *Check) InstanceName() string { + return c.instName +} + +func (c *Check) Init(cfg *config.Map) error { + cfg.Bool("debug", true, false, &c.log.Debug) + cfg.Bool("enforce_early", true, false, &c.enforceEarly) + cfg.Custom("none_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.noneAction) + cfg.Custom("neutral_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.neutralAction) + cfg.Custom("fail_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{Quarantine: true}, nil + }, modconfig.FailActionDirective, &c.failAction) + cfg.Custom("softfail_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.softfailAction) + cfg.Custom("permerr_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.permerrAction) + cfg.Custom("temperr_action", false, false, + func() (interface{}, error) { + return modconfig.FailAction{}, nil + }, modconfig.FailActionDirective, &c.temperrAction) + _, err := cfg.Process() + if err != nil { + return err + } + + return nil +} + +type spfRes struct { + res spf.Result + err error +} + +type state struct { + c *Check + msgMeta *module.MsgMetadata + spfFetch chan spfRes + log log.Logger + + skip bool +} + +func (c *Check) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &state{ + c: c, + msgMeta: msgMeta, + spfFetch: make(chan spfRes, 1), + log: target.DeliveryLogger(c.log, msgMeta), + }, nil +} + +func (s *state) spfResult(res spf.Result, err error) module.CheckResult { + _, fromDomain, _ := address.Split(s.msgMeta.OriginalFrom) + spfAuth := &authres.SPFResult{ + Value: authres.ResultNone, + Helo: s.msgMeta.Conn.Hostname, + From: fromDomain, + } + + if err != nil { + spfAuth.Reason = err.Error() + } else if res == spf.None { + spfAuth.Reason = "no policy" + } + + switch res { + case spf.None: + spfAuth.Value = authres.ResultNone + return s.c.noneAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 23}, + Message: "No SPF policy", + CheckName: modName, + Err: err, + }, + AuthResult: []authres.Result{spfAuth}, + }) + case spf.Neutral: + spfAuth.Value = authres.ResultNeutral + return s.c.neutralAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 23}, + Message: "Neutral SPF result is not permitted", + CheckName: modName, + Err: err, + }, + AuthResult: []authres.Result{spfAuth}, + }) + case spf.Pass: + spfAuth.Value = authres.ResultPass + return module.CheckResult{AuthResult: []authres.Result{spfAuth}} + case spf.Fail: + spfAuth.Value = authres.ResultFail + return s.c.failAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 23}, + Message: "SPF authentication failed", + CheckName: modName, + Err: err, + }, + AuthResult: []authres.Result{spfAuth}, + }) + case spf.SoftFail: + spfAuth.Value = authres.ResultSoftFail + return s.c.softfailAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 23}, + Message: "SPF authentication soft-failed", + CheckName: modName, + Err: err, + }, + AuthResult: []authres.Result{spfAuth}, + }) + case spf.TempError: + spfAuth.Value = authres.ResultTempError + return s.c.temperrAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 23}, + Message: "SPF authentication failed with a temporary error", + CheckName: modName, + Err: err, + }, + AuthResult: []authres.Result{spfAuth}, + }) + case spf.PermError: + spfAuth.Value = authres.ResultPermError + return s.c.permerrAction.Apply(module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 23}, + Message: "SPF authentication failed with a permanent error", + CheckName: modName, + Err: err, + }, + AuthResult: []authres.Result{spfAuth}, + }) + } + + return module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{4, 7, 23}, + Message: fmt.Sprintf("Unknown SPF status: %s", res), + CheckName: modName, + Err: err, + }, + AuthResult: []authres.Result{spfAuth}, + } +} + +func (s *state) relyOnDMARC(ctx context.Context, hdr textproto.Header) bool { + fromDomain, err := maddydmarc.ExtractFromDomain(hdr) + if err != nil { + s.log.Error("DMARC domains extract", err) + return false + } + + policyDomain, record, err := maddydmarc.FetchRecord(ctx, s.c.resolver, fromDomain) + if err != nil { + s.log.Error("DMARC fetch", err, "from_domain", fromDomain) + return false + } + if record == nil { + return false + } + + policy := record.Policy + // We check for subdomain using non-equality since fromDomain is either the + // subdomain of policyDomain or policyDomain itself (due to the way + // FetchRecord handles it). + if !dns.Equal(policyDomain, fromDomain) && record.SubdomainPolicy != "" { + policy = record.SubdomainPolicy + } + + return policy != dmarc.PolicyNone +} + +func prepareMailFrom(from string) (string, error) { + // INTERNATIONALIZATION: RFC 8616, Section 4 + // Hostname is already in A-labels per SMTPUTF8 requirement. + // MAIL FROM domain should be converted to A-labels before doing + // anything. + fromMbox, fromDomain, err := address.Split(from) + if err != nil { + return "", &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 1, 7}, + Message: "Malformed address", + CheckName: "spf", + } + } + fromDomain, err = idna.ToASCII(fromDomain) + if err != nil { + return "", &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 1, 7}, + Message: "Malformed address", + CheckName: "spf", + } + } + + // %{s} and %{l} do not match anything if it is non-ASCII. + // Since spf lib does not seem to care, strip it. + if !address.IsASCII(fromMbox) { + fromMbox = "" + } + + return fromMbox + "@" + dns.FQDN(fromDomain), nil +} + +func (s *state) CheckConnection(ctx context.Context) module.CheckResult { + defer trace.StartRegion(ctx, "check.spf/CheckConnection").End() + + if s.msgMeta.Conn == nil { + s.skip = true + s.log.Println("locally generated message, skipping") + return module.CheckResult{} + } + + ip, ok := s.msgMeta.Conn.RemoteAddr.(*net.TCPAddr) + if !ok { + s.skip = true + s.log.Println("non-IP SrcAddr") + return module.CheckResult{} + } + + mailFromOriginal := s.msgMeta.OriginalFrom + if mailFromOriginal == "" { + // RFC 7208 Section 2.4. + // >When the reverse-path is null, this document + // >defines the "MAIL FROM" identity to be the mailbox composed of the + // >local-part "postmaster" and the "HELO" identity (which might or might + // >not have been checked separately before). + mailFromOriginal = "postmaster@" + s.msgMeta.Conn.Hostname + } + + mailFrom, err := prepareMailFrom(mailFromOriginal) + if err != nil { + s.skip = true + return module.CheckResult{ + Reason: err, + Reject: true, + } + } + + if s.c.enforceEarly { + res, err := spf.CheckHostWithSender(ip.IP, + dns.FQDN(s.msgMeta.Conn.Hostname), mailFrom, + spf.WithContext(ctx), spf.WithResolver(s.c.resolver)) + s.log.Debugf("result: %s (%v)", res, err) + return s.spfResult(res, err) + } + + // We start evaluation in parallel to other message processing, + // once we get the body, we fetch DMARC policy and see if it exists + // and not p=none. In that case, we rely on DMARC alignment to define result. + // Otherwise, we take action based on SPF only. + + go func() { + defer func() { + if err := recover(); err != nil { + stack := debug.Stack() + log.Printf("panic during spf.CheckHostWithSender: %v\n%s", err, stack) + close(s.spfFetch) + } + }() + + defer trace.StartRegion(ctx, "check.spf/CheckConnection (Async)").End() + + res, err := spf.CheckHostWithSender(ip.IP, dns.FQDN(s.msgMeta.Conn.Hostname), mailFrom, + spf.WithContext(ctx), spf.WithResolver(s.c.resolver)) + s.log.Debugf("result: %s (%v)", res, err) + s.spfFetch <- spfRes{res, err} + }() + + return module.CheckResult{} +} + +func (s *state) CheckSender(ctx context.Context, mailFrom string) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckRcpt(ctx context.Context, rcptTo string) module.CheckResult { + return module.CheckResult{} +} + +func (s *state) CheckBody(ctx context.Context, header textproto.Header, body buffer.Buffer) module.CheckResult { + if s.c.enforceEarly || s.skip { + // Already applied in CheckConnection. + return module.CheckResult{} + } + + defer trace.StartRegion(ctx, "check.spf/CheckBody").End() + + res, ok := <-s.spfFetch + if !ok { + return module.CheckResult{ + Reject: true, + Reason: exterrors.WithTemporary( + exterrors.WithFields(errors.New("panic recovered"), map[string]interface{}{ + "check": "spf", + "smtp_msg": "Internal error during policy check", + }), + true, + ), + } + } + if s.relyOnDMARC(ctx, header) { + if res.res != spf.Pass { + s.log.Msg("deferring action due to a DMARC policy", "result", res.res, "err", res.err) + } else { + s.log.DebugMsg("deferring action due to a DMARC policy", "result", res.res, "err", res.err) + } + + checkRes := s.spfResult(res.res, res.err) + checkRes.Quarantine = false + checkRes.Reject = false + return checkRes + } + + return s.spfResult(res.res, res.err) +} + +func (s *state) Close() error { + return nil +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/check/stateless_check.go b/internal/check/stateless_check.go new file mode 100644 index 0000000..4c5d2d5 --- /dev/null +++ b/internal/check/stateless_check.go @@ -0,0 +1,202 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package check + +import ( + "context" + "fmt" + "runtime/trace" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +type ( + StatelessCheckContext struct { + // Embedded context.Context value, used for tracing, cancellation and + // timeouts. + context.Context + + // Resolver that should be used by the check for DNS queries. + Resolver dns.Resolver + + MsgMeta *module.MsgMetadata + + // Logger that should be used by the check for logging, note that it is + // already wrapped to append Msg ID to all messages so check code + // should not do the same. + Logger log.Logger + } + FuncConnCheck func(checkContext StatelessCheckContext) module.CheckResult + FuncSenderCheck func(checkContext StatelessCheckContext, mailFrom string) module.CheckResult + FuncRcptCheck func(checkContext StatelessCheckContext, rcptTo string) module.CheckResult + FuncBodyCheck func(checkContext StatelessCheckContext, header textproto.Header, body buffer.Buffer) module.CheckResult +) + +type statelessCheck struct { + modName string + instName string + resolver dns.Resolver + logger log.Logger + + // One used by Init if config option is not passed by a user. + defaultFailAction modconfig.FailAction + // The actual fail action that should be applied. + failAction modconfig.FailAction + + connCheck FuncConnCheck + senderCheck FuncSenderCheck + rcptCheck FuncRcptCheck + bodyCheck FuncBodyCheck +} + +type statelessCheckState struct { + c *statelessCheck + msgMeta *module.MsgMetadata +} + +func (s *statelessCheckState) String() string { + return s.c.modName + ":" + s.c.instName +} + +func (s *statelessCheckState) CheckConnection(ctx context.Context) module.CheckResult { + if s.c.connCheck == nil { + return module.CheckResult{} + } + defer trace.StartRegion(ctx, s.c.modName+"/CheckConnection").End() + + originalRes := s.c.connCheck(StatelessCheckContext{ + Context: ctx, + Resolver: s.c.resolver, + MsgMeta: s.msgMeta, + Logger: target.DeliveryLogger(s.c.logger, s.msgMeta), + }) + return s.c.failAction.Apply(originalRes) +} + +func (s *statelessCheckState) CheckSender(ctx context.Context, mailFrom string) module.CheckResult { + if s.c.senderCheck == nil { + return module.CheckResult{} + } + defer trace.StartRegion(ctx, s.c.modName+"/CheckSender").End() + + originalRes := s.c.senderCheck(StatelessCheckContext{ + Context: ctx, + Resolver: s.c.resolver, + MsgMeta: s.msgMeta, + Logger: target.DeliveryLogger(s.c.logger, s.msgMeta), + }, mailFrom) + return s.c.failAction.Apply(originalRes) +} + +func (s *statelessCheckState) CheckRcpt(ctx context.Context, rcptTo string) module.CheckResult { + if s.c.rcptCheck == nil { + return module.CheckResult{} + } + defer trace.StartRegion(ctx, s.c.modName+"/CheckRcpt").End() + + originalRes := s.c.rcptCheck(StatelessCheckContext{ + Context: ctx, + Resolver: s.c.resolver, + MsgMeta: s.msgMeta, + Logger: target.DeliveryLogger(s.c.logger, s.msgMeta), + }, rcptTo) + return s.c.failAction.Apply(originalRes) +} + +func (s *statelessCheckState) CheckBody(ctx context.Context, header textproto.Header, body buffer.Buffer) module.CheckResult { + if s.c.bodyCheck == nil { + return module.CheckResult{} + } + defer trace.StartRegion(ctx, s.c.modName+"/CheckBody").End() + + originalRes := s.c.bodyCheck(StatelessCheckContext{ + Context: ctx, + Resolver: s.c.resolver, + MsgMeta: s.msgMeta, + Logger: target.DeliveryLogger(s.c.logger, s.msgMeta), + }, header, body) + return s.c.failAction.Apply(originalRes) +} + +func (s *statelessCheckState) Close() error { + return nil +} + +func (c *statelessCheck) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + return &statelessCheckState{ + c: c, + msgMeta: msgMeta, + }, nil +} + +func (c *statelessCheck) Init(cfg *config.Map) error { + cfg.Bool("debug", true, false, &c.logger.Debug) + cfg.Custom("fail_action", false, false, + func() (interface{}, error) { + return c.defaultFailAction, nil + }, modconfig.FailActionDirective, &c.failAction) + _, err := cfg.Process() + return err +} + +func (c *statelessCheck) Name() string { + return c.modName +} + +func (c *statelessCheck) InstanceName() string { + return c.instName +} + +// RegisterStatelessCheck is helper function to create stateless message check modules +// that run one simple check during one stage. +// +// It creates the module and its instance with the specified name that implement module.Check interface +// and runs passed functions when corresponding module.CheckState methods are called. +// +// Note about CheckResult that is returned by the functions: +// StatelessCheck supports different action types based on the user configuration, but the particular check +// code doesn't need to know about it. It should assume that it is always "Reject" and hence it should +// populate Reason field of the result object with the relevant error description. +func RegisterStatelessCheck(name string, defaultFailAction modconfig.FailAction, connCheck FuncConnCheck, senderCheck FuncSenderCheck, rcptCheck FuncRcptCheck, bodyCheck FuncBodyCheck) { + module.Register(name, func(modName, instName string, aliases, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, fmt.Errorf("%s: inline arguments are not used", modName) + } + return &statelessCheck{ + modName: modName, + instName: instName, + resolver: dns.DefaultResolver(), + logger: log.Logger{Name: modName}, + + defaultFailAction: defaultFailAction, + + connCheck: connCheck, + senderCheck: senderCheck, + rcptCheck: rcptCheck, + bodyCheck: bodyCheck, + }, nil + }) +} diff --git a/internal/cli/app.go b/internal/cli/app.go new file mode 100644 index 0000000..5910896 --- /dev/null +++ b/internal/cli/app.go @@ -0,0 +1,113 @@ +package maddycli + +import ( + "fmt" + "os" + "strings" + + "github.com/foxcpp/maddy/framework/log" + "github.com/urfave/cli/v2" +) + +var app *cli.App + +func init() { + app = cli.NewApp() + app.Usage = "composable all-in-one mail server" + app.Description = `Maddy is Mail Transfer agent (MTA), Mail Delivery Agent (MDA), Mail Submission +Agent (MSA), IMAP server and a set of other essential protocols/schemes +necessary to run secure email server implemented in one executable. + +This executable can be used to start the server ('run') and to manipulate +databases used by it (all other subcommands). +` + app.Authors = []*cli.Author{ + { + Name: "Maddy Mail Server maintainers & contributors", + Email: "~foxcpp/maddy@lists.sr.ht", + }, + } + app.ExitErrHandler = func(c *cli.Context, err error) { + cli.HandleExitCoder(err) + } + app.EnableBashCompletion = true + app.Commands = []*cli.Command{ + { + Name: "generate-man", + Hidden: true, + Action: func(c *cli.Context) error { + man, err := app.ToMan() + if err != nil { + return err + } + fmt.Println(man) + return nil + }, + }, + { + Name: "generate-fish-completion", + Hidden: true, + Action: func(c *cli.Context) error { + cp, err := app.ToFishCompletion() + if err != nil { + return err + } + fmt.Println(cp) + return nil + }, + }, + } +} + +func AddGlobalFlag(f cli.Flag) { + app.Flags = append(app.Flags, f) +} + +func AddSubcommand(cmd *cli.Command) { + app.Commands = append(app.Commands, cmd) + + if cmd.Name == "run" { + // Backward compatibility hack to start the server as just ./maddy + // Needs to be done here so we will register all known flags with + // stdlib before Run is called. + app.Action = func(c *cli.Context) error { + log.Println("WARNING: Starting server not via 'maddy run' is deprecated and will stop working in the next version") + return cmd.Action(c) + } + app.Flags = append(app.Flags, cmd.Flags...) + } +} + +// RunWithoutExit is like Run but returns exit code instead of calling os.Exit +// To be used in maddy.cover. +func RunWithoutExit() int { + code := 0 + + cli.OsExiter = func(c int) { code = c } + defer func() { + cli.OsExiter = os.Exit + }() + + Run() + + return code +} + +func Run() { + mapStdlibFlags(app) + + // Actual entry point is registered in maddy.go. + + // Print help when called via maddyctl executable. To be removed + // once backward compatibility hack for 'maddy run' is removed too. + if strings.Contains(os.Args[0], "maddyctl") && len(os.Args) == 1 { + if err := app.Run([]string{os.Args[0], "help"}); err != nil { + log.DefaultLogger.Error("app.Run failed", err) + } + return + } + + if err := app.Run(os.Args); err != nil { + log.DefaultLogger.Error("app.Run failed", err) + } +} diff --git a/internal/cli/clitools/clitools.go b/internal/cli/clitools/clitools.go new file mode 100644 index 0000000..2da6998 --- /dev/null +++ b/internal/cli/clitools/clitools.go @@ -0,0 +1,118 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package clitools + +import ( + "bufio" + "errors" + "fmt" + "os" +) + +var stdinScanner = bufio.NewScanner(os.Stdin) + +func Confirmation(prompt string, def bool) bool { + selection := "y/N" + if def { + selection = "Y/n" + } + + fmt.Fprintf(os.Stderr, "%s [%s]: ", prompt, selection) + if !stdinScanner.Scan() { + fmt.Fprintln(os.Stderr, stdinScanner.Err()) + return false + } + + switch stdinScanner.Text() { + case "Y", "y": + return true + case "N", "n": + return false + default: + return def + } +} + +func readPass(tty *os.File, output []byte) ([]byte, error) { + cursor := output[0:1] + readen := 0 + for { + n, err := tty.Read(cursor) + if n != 1 { + return nil, errors.New("ReadPassword: invalid read size when not in canonical mode") + } + if err != nil { + return nil, errors.New("ReadPassword: " + err.Error()) + } + if cursor[0] == '\n' { + break + } + // Esc or Ctrl+D or Ctrl+C. + if cursor[0] == '\x1b' || cursor[0] == '\x04' || cursor[0] == '\x03' { + return nil, errors.New("ReadPassword: prompt rejected") + } + if cursor[0] == '\x7F' /* DEL */ { + if readen != 0 { + readen-- + cursor = output[readen : readen+1] + } + continue + } + + if readen == cap(output) { + return nil, errors.New("ReadPassword: too long password") + } + + readen++ + cursor = output[readen : readen+1] + } + + return output[0:readen], nil +} + +func ReadPassword(prompt string) (string, error) { + termios, err := TurnOnRawIO(os.Stdin) + hiddenPass := true + if err != nil { + hiddenPass = false + fmt.Fprintln(os.Stderr, "Failed to disable terminal output:", err) + } + + // There is no meaningful way to handle error here. + //nolint:errcheck + defer TcSetAttr(os.Stdin.Fd(), &termios) + + fmt.Fprintf(os.Stderr, "%s: ", prompt) + + if hiddenPass { + buf := make([]byte, 512) + buf, err = readPass(os.Stdin, buf) + if err != nil { + return "", err + } + fmt.Println() + + return string(buf), nil + } + if !stdinScanner.Scan() { + return "", stdinScanner.Err() + } + + return stdinScanner.Text(), nil +} diff --git a/internal/cli/clitools/termios.go b/internal/cli/clitools/termios.go new file mode 100644 index 0000000..cf817d1 --- /dev/null +++ b/internal/cli/clitools/termios.go @@ -0,0 +1,82 @@ +//go:build linux +// +build linux + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package clitools + +// Copied from github.com/foxcpp/ttyprompt +// Commit 087a574, terminal/termios.go + +import ( + "errors" + "os" + "syscall" + "unsafe" +) + +type Termios struct { + Iflag uint32 + Oflag uint32 + Cflag uint32 + Lflag uint32 + Cc [20]byte + Ispeed uint32 + Ospeed uint32 +} + +/* +TurnOnRawIO sets flags suitable for raw I/O (no echo, per-character input, etc) +and returns original flags. +*/ +func TurnOnRawIO(tty *os.File) (orig Termios, err error) { + termios, err := TcGetAttr(tty.Fd()) + if err != nil { + return Termios{}, errors.New("TurnOnRawIO: failed to get flags: " + err.Error()) + } + termiosOrig := *termios + + termios.Lflag &^= syscall.ECHO + termios.Lflag &^= syscall.ICANON + termios.Iflag &^= syscall.IXON + termios.Lflag &^= syscall.ISIG + termios.Iflag |= syscall.IUTF8 + err = TcSetAttr(tty.Fd(), termios) + if err != nil { + return Termios{}, errors.New("TurnOnRawIO: flags to set flags: " + err.Error()) + } + return termiosOrig, nil +} + +func TcSetAttr(fd uintptr, termios *Termios) error { + _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCSETS, uintptr(unsafe.Pointer(termios))) + if err != 0 { + return err + } + return nil +} + +func TcGetAttr(fd uintptr) (*Termios, error) { + termios := &Termios{} + _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(termios))) + if err != 0 { + return nil, err + } + return termios, nil +} diff --git a/internal/cli/clitools/termios_stub.go b/internal/cli/clitools/termios_stub.go new file mode 100644 index 0000000..03397fa --- /dev/null +++ b/internal/cli/clitools/termios_stub.go @@ -0,0 +1,49 @@ +//go:build !linux +// +build !linux + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package clitools + +import ( + "errors" + "os" +) + +type Termios struct { + Iflag uint32 + Oflag uint32 + Cflag uint32 + Lflag uint32 + Cc [20]byte + Ispeed uint32 + Ospeed uint32 +} + +func TurnOnRawIO(tty *os.File) (orig Termios, err error) { + return Termios{}, errors.New("not implemented") +} + +func TcSetAttr(fd uintptr, termios *Termios) error { + return errors.New("not implemented") +} + +func TcGetAttr(fd uintptr) (*Termios, error) { + return nil, errors.New("not implemented") +} diff --git a/internal/cli/ctl/appendlimit.go b/internal/cli/ctl/appendlimit.go new file mode 100644 index 0000000..ce0e3c4 --- /dev/null +++ b/internal/cli/ctl/appendlimit.go @@ -0,0 +1,79 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package ctl + +import ( + "fmt" + + imapbackend "github.com/emersion/go-imap/backend" + "github.com/foxcpp/maddy/framework/module" + "github.com/urfave/cli/v2" +) + +// Copied from go-imap-backend-tests. + +// AppendLimitUser is extension for backend.User interface which allows to +// set append limit value for testing and administration purposes. +type AppendLimitUser interface { + imapbackend.AppendLimitUser + + // SetMessageLimit sets new value for limit. + // nil pointer means no limit. + SetMessageLimit(val *uint32) error +} + +func imapAcctAppendlimit(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + userAL, ok := u.(AppendLimitUser) + if !ok { + return cli.Exit("Error: module.Storage does not support per-user append limit", 2) + } + + if ctx.IsSet("value") { + val := ctx.Int("value") + + var err error + if val == -1 { + err = userAL.SetMessageLimit(nil) + } else { + val32 := uint32(val) + err = userAL.SetMessageLimit(&val32) + } + if err != nil { + return err + } + } else { + lim := userAL.CreateMessageLimit() + if lim == nil { + fmt.Println("No limit") + } else { + fmt.Println(*lim) + } + } + + return nil +} diff --git a/internal/cli/ctl/hash.go b/internal/cli/ctl/hash.go new file mode 100644 index 0000000..5effdf7 --- /dev/null +++ b/internal/cli/ctl/hash.go @@ -0,0 +1,139 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package ctl + +import ( + "fmt" + "os" + "strings" + + "github.com/foxcpp/maddy/internal/auth/pass_table" + maddycli "github.com/foxcpp/maddy/internal/cli" + clitools2 "github.com/foxcpp/maddy/internal/cli/clitools" + "github.com/urfave/cli/v2" + "golang.org/x/crypto/bcrypt" +) + +func init() { + maddycli.AddSubcommand( + &cli.Command{ + Name: "hash", + Usage: "Generate password hashes for use with pass_table", + Action: hashCommand, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "password", + Aliases: []string{"p"}, + Usage: "Use `PASSWORD instead of reading password from stdin\n\t\tWARNING: Provided only for debugging convenience. Don't leave your passwords in shell history!", + }, + &cli.StringFlag{ + Name: "hash", + Usage: "Use specified hash algorithm", + Value: "bcrypt", + }, + &cli.IntFlag{ + Name: "bcrypt-cost", + Usage: "Specify bcrypt cost value", + Value: bcrypt.DefaultCost, + }, + &cli.IntFlag{ + Name: "argon2-time", + Usage: "Time factor for Argon2id", + Value: 3, + }, + &cli.IntFlag{ + Name: "argon2-memory", + Usage: "Memory in KiB to use for Argon2id", + Value: 1024, + }, + &cli.IntFlag{ + Name: "argon2-threads", + Usage: "Threads to use for Argon2id", + Value: 1, + }, + }, + }) +} + +func hashCommand(ctx *cli.Context) error { + hashFunc := ctx.String("hash") + if hashFunc == "" { + hashFunc = pass_table.DefaultHash + } + + hashCompute := pass_table.HashCompute[hashFunc] + if hashCompute == nil { + var funcs []string + for k := range pass_table.HashCompute { + funcs = append(funcs, k) + } + + return cli.Exit(fmt.Sprintf("Error: Unknown hash function, available: %s", strings.Join(funcs, ", ")), 2) + } + + opts := pass_table.HashOpts{ + BcryptCost: bcrypt.DefaultCost, + Argon2Memory: 1024, + Argon2Time: 2, + Argon2Threads: 1, + } + if ctx.IsSet("bcrypt-cost") { + if ctx.Int("bcrypt-cost") > bcrypt.MaxCost { + return cli.Exit("Error: too big bcrypt cost", 2) + } + if ctx.Int("bcrypt-cost") < bcrypt.MinCost { + return cli.Exit("Error: too small bcrypt cost", 2) + } + opts.BcryptCost = ctx.Int("bcrypt-cost") + } + if ctx.IsSet("argon2-memory") { + opts.Argon2Memory = uint32(ctx.Int("argon2-memory")) + } + if ctx.IsSet("argon2-time") { + opts.Argon2Time = uint32(ctx.Int("argon2-time")) + } + if ctx.IsSet("argon2-threads") { + opts.Argon2Threads = uint8(ctx.Int("argon2-threads")) + } + + var pass string + if ctx.IsSet("password") { + pass = ctx.String("password") + } else { + var err error + pass, err = clitools2.ReadPassword("Password") + if err != nil { + return err + } + } + + if pass == "" { + fmt.Fprintln(os.Stderr, "WARNING: This is the hash of an empty string") + } + if strings.TrimSpace(pass) != pass { + fmt.Fprintln(os.Stderr, "WARNING: There is leading/trailing whitespace in the string") + } + + hash, err := hashCompute(opts, pass) + if err != nil { + return err + } + fmt.Println(hashFunc + ":" + hash) + return nil +} diff --git a/internal/cli/ctl/imap.go b/internal/cli/ctl/imap.go new file mode 100644 index 0000000..ea8f820 --- /dev/null +++ b/internal/cli/ctl/imap.go @@ -0,0 +1,893 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package ctl + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/emersion/go-imap" + imapsql "github.com/foxcpp/go-imap-sql" + "github.com/foxcpp/maddy/framework/module" + maddycli "github.com/foxcpp/maddy/internal/cli" + clitools2 "github.com/foxcpp/maddy/internal/cli/clitools" + "github.com/urfave/cli/v2" +) + +func init() { + maddycli.AddSubcommand( + &cli.Command{ + Name: "imap-mboxes", + Usage: "IMAP mailboxes (folders) management", + Subcommands: []*cli.Command{ + { + Name: "list", + Usage: "Show mailboxes of user", + ArgsUsage: "USERNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "subscribed", + Aliases: []string{"s"}, + Usage: "List only subscribed mailboxes", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return mboxesList(be, ctx) + }, + }, + { + Name: "create", + Usage: "Create mailbox", + ArgsUsage: "USERNAME NAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.StringFlag{ + Name: "special", + Usage: "Set SPECIAL-USE attribute on mailbox; valid values: archive, drafts, junk, sent, trash", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return mboxesCreate(be, ctx) + }, + }, + { + Name: "remove", + Usage: "Remove mailbox", + Description: "WARNING: All contents of mailbox will be irrecoverably lost.", + ArgsUsage: "USERNAME MAILBOX", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "yes", + Aliases: []string{"y"}, + Usage: "Don't ask for confirmation", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return mboxesRemove(be, ctx) + }, + }, + { + Name: "rename", + Usage: "Rename mailbox", + Description: "Rename may cause unexpected failures on client-side so be careful.", + ArgsUsage: "USERNAME OLDNAME NEWNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return mboxesRename(be, ctx) + }, + }, + }, + }) + maddycli.AddSubcommand(&cli.Command{ + Name: "imap-msgs", + Usage: "IMAP messages management", + Subcommands: []*cli.Command{ + { + Name: "add", + Usage: "Add message to mailbox", + ArgsUsage: "USERNAME MAILBOX", + Description: "Reads message body (with headers) from stdin. Prints UID of created message on success.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.StringSliceFlag{ + Name: "flag", + Aliases: []string{"f"}, + Usage: "Add flag to message. Can be specified multiple times", + }, + &cli.TimestampFlag{ + Layout: time.RFC3339, + Name: "date", + Aliases: []string{"d"}, + Usage: "Set internal date value to specified one in ISO 8601 format (2006-01-02T15:04:05Z07:00)", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsAdd(be, ctx) + }, + }, + { + Name: "add-flags", + Usage: "Add flags to messages", + ArgsUsage: "USERNAME MAILBOX SEQ FLAGS...", + Description: "Add flags to all messages matched by SEQ.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQSET instead of sequence numbers", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsFlags(be, ctx) + }, + }, + { + Name: "rem-flags", + Usage: "Remove flags from messages", + ArgsUsage: "USERNAME MAILBOX SEQ FLAGS...", + Description: "Remove flags from all messages matched by SEQ.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQSET instead of sequence numbers", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsFlags(be, ctx) + }, + }, + { + Name: "set-flags", + Usage: "Set flags on messages", + ArgsUsage: "USERNAME MAILBOX SEQ FLAGS...", + Description: "Set flags on all messages matched by SEQ.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQSET instead of sequence numbers", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsFlags(be, ctx) + }, + }, + { + Name: "remove", + Usage: "Remove messages from mailbox", + ArgsUsage: "USERNAME MAILBOX SEQSET", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid,u", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQSET instead of sequence numbers", + }, + &cli.BoolFlag{ + Name: "yes", + Aliases: []string{"y"}, + Usage: "Don't ask for confirmation", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsRemove(be, ctx) + }, + }, + { + Name: "copy", + Usage: "Copy messages between mailboxes", + Description: "Note: You can't copy between mailboxes of different users. APPENDLIMIT of target mailbox is not enforced.", + ArgsUsage: "USERNAME SRCMAILBOX SEQSET TGTMAILBOX", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQSET instead of sequence numbers", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsCopy(be, ctx) + }, + }, + { + Name: "move", + Usage: "Move messages between mailboxes", + Description: "Note: You can't move between mailboxes of different users. APPENDLIMIT of target mailbox is not enforced.", + ArgsUsage: "USERNAME SRCMAILBOX SEQSET TGTMAILBOX", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQSET instead of sequence numbers", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsMove(be, ctx) + }, + }, + { + Name: "list", + Usage: "List messages in mailbox", + Description: "If SEQSET is specified - only show messages that match it.", + ArgsUsage: "USERNAME MAILBOX [SEQSET]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQSET instead of sequence numbers", + }, + &cli.BoolFlag{ + Name: "full,f", + Aliases: []string{"f"}, + Usage: "Show entire envelope and all server meta-data", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsList(be, ctx) + }, + }, + { + Name: "dump", + Usage: "Dump message body", + Description: "If passed SEQ matches multiple messages - they will be joined.", + ArgsUsage: "USERNAME MAILBOX SEQ", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "uid", + Aliases: []string{"u"}, + Usage: "Use UIDs for SEQ instead of sequence numbers", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return msgsDump(be, ctx) + }, + }, + }, + }) +} + +func FormatAddress(addr *imap.Address) string { + return fmt.Sprintf("%s <%s@%s>", addr.PersonalName, addr.MailboxName, addr.HostName) +} + +func FormatAddressList(addrs []*imap.Address) string { + res := make([]string, 0, len(addrs)) + for _, addr := range addrs { + res = append(res, FormatAddress(addr)) + } + return strings.Join(res, ", ") +} + +func mboxesList(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + mboxes, err := u.ListMailboxes(ctx.Bool("subscribed,s")) + if err != nil { + return err + } + + if len(mboxes) == 0 && !ctx.Bool("quiet") { + fmt.Fprintln(os.Stderr, "No mailboxes.") + } + + for _, info := range mboxes { + if len(info.Attributes) != 0 { + fmt.Print(info.Name, "\t", info.Attributes, "\n") + } else { + fmt.Println(info.Name) + } + } + + return nil +} + +func mboxesCreate(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + name := ctx.Args().Get(1) + if name == "" { + return cli.Exit("Error: NAME is required", 2) + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + if ctx.IsSet("special") { + attr := "\\" + strings.Title(ctx.String("special")) //nolint:staticcheck + // (nolint) strings.Title is perfectly fine there since special mailbox tags will never use Unicode. + + suu, ok := u.(SpecialUseUser) + if !ok { + return cli.Exit("Error: storage backend does not support SPECIAL-USE IMAP extension", 2) + } + + return suu.CreateMailboxSpecial(name, attr) + } + + return u.CreateMailbox(name) +} + +func mboxesRemove(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + name := ctx.Args().Get(1) + if name == "" { + return cli.Exit("Error: NAME is required", 2) + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + if !ctx.Bool("yes") { + status, err := u.Status(name, []imap.StatusItem{imap.StatusMessages}) + if err != nil { + return err + } + + if status.Messages != 0 { + fmt.Fprintf(os.Stderr, "Mailbox %s contains %d messages.\n", name, status.Messages) + } + + if !clitools2.Confirmation("Are you sure you want to delete that mailbox?", false) { + return errors.New("Cancelled") + } + } + + return u.DeleteMailbox(name) +} + +func mboxesRename(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + oldName := ctx.Args().Get(1) + if oldName == "" { + return cli.Exit("Error: OLDNAME is required", 2) + } + newName := ctx.Args().Get(2) + if newName == "" { + return cli.Exit("Error: NEWNAME is required", 2) + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + return u.RenameMailbox(oldName, newName) +} + +func msgsAdd(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + name := ctx.Args().Get(1) + if name == "" { + return cli.Exit("Error: MAILBOX is required", 2) + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + flags := ctx.StringSlice("flag") + if flags == nil { + flags = []string{} + } + + date := time.Now() + if ctx.IsSet("date") { + date = *ctx.Timestamp("date") + } + + buf := bytes.Buffer{} + if _, err := io.Copy(&buf, os.Stdin); err != nil { + return err + } + + if buf.Len() == 0 { + return cli.Exit("Error: Empty message, refusing to continue", 2) + } + + status, err := u.Status(name, []imap.StatusItem{imap.StatusUidNext}) + if err != nil { + return err + } + + if err := u.CreateMessage(name, flags, date, &buf, nil); err != nil { + return err + } + + // TODO: Use APPENDUID + fmt.Println(status.UidNext) + + return nil +} + +func msgsRemove(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + name := ctx.Args().Get(1) + if name == "" { + return cli.Exit("Error: MAILBOX is required", 2) + } + seqset := ctx.Args().Get(2) + if seqset == "" { + return cli.Exit("Error: SEQSET is required", 2) + } + + if !ctx.Bool("uid") { + fmt.Fprintln(os.Stderr, "WARNING: --uid=true will be the default in 0.7") + } + + seq, err := imap.ParseSeqSet(seqset) + if err != nil { + return err + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + _, mbox, err := u.GetMailbox(name, true, nil) + if err != nil { + return err + } + + if !ctx.Bool("yes") { + if !clitools2.Confirmation("Are you sure you want to delete these messages?", false) { + return errors.New("Cancelled") + } + } + + mboxB := mbox.(*imapsql.Mailbox) + return mboxB.DelMessages(ctx.Bool("uid"), seq) +} + +func msgsCopy(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + srcName := ctx.Args().Get(1) + if srcName == "" { + return cli.Exit("Error: SRCMAILBOX is required", 2) + } + seqset := ctx.Args().Get(2) + if seqset == "" { + return cli.Exit("Error: SEQSET is required", 2) + } + tgtName := ctx.Args().Get(3) + if tgtName == "" { + return cli.Exit("Error: TGTMAILBOX is required", 2) + } + + if !ctx.Bool("uid") { + fmt.Fprintln(os.Stderr, "WARNING: --uid=true will be the default in 0.7") + } + + seq, err := imap.ParseSeqSet(seqset) + if err != nil { + return err + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + _, srcMbox, err := u.GetMailbox(srcName, true, nil) + if err != nil { + return err + } + + return srcMbox.CopyMessages(ctx.Bool("uid"), seq, tgtName) +} + +func msgsMove(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + srcName := ctx.Args().Get(1) + if srcName == "" { + return cli.Exit("Error: SRCMAILBOX is required", 2) + } + seqset := ctx.Args().Get(2) + if seqset == "" { + return cli.Exit("Error: SEQSET is required", 2) + } + tgtName := ctx.Args().Get(3) + if tgtName == "" { + return cli.Exit("Error: TGTMAILBOX is required", 2) + } + + if !ctx.Bool("uid") { + fmt.Fprintln(os.Stderr, "WARNING: --uid=true will be the default in 0.7") + } + + seq, err := imap.ParseSeqSet(seqset) + if err != nil { + return err + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + _, srcMbox, err := u.GetMailbox(srcName, true, nil) + if err != nil { + return err + } + + moveMbox := srcMbox.(*imapsql.Mailbox) + + return moveMbox.MoveMessages(ctx.Bool("uid"), seq, tgtName) +} + +func msgsList(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + mboxName := ctx.Args().Get(1) + if mboxName == "" { + return cli.Exit("Error: MAILBOX is required", 2) + } + seqset := ctx.Args().Get(2) + uid := ctx.Bool("uid") + if seqset == "" { + seqset = "1:*" + uid = true + } else if !uid { + fmt.Fprintln(os.Stderr, "WARNING: --uid=true will be the default in 0.7") + } + + seq, err := imap.ParseSeqSet(seqset) + if err != nil { + return err + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + _, mbox, err := u.GetMailbox(mboxName, true, nil) + if err != nil { + return err + } + + ch := make(chan *imap.Message, 10) + go func() { + err = mbox.ListMessages(uid, seq, []imap.FetchItem{imap.FetchEnvelope, imap.FetchInternalDate, imap.FetchRFC822Size, imap.FetchFlags, imap.FetchUid}, ch) + }() + + for msg := range ch { + if !ctx.Bool("full") { + fmt.Printf("UID %d: %s - %s\n %v, %v\n\n", msg.Uid, FormatAddressList(msg.Envelope.From), msg.Envelope.Subject, msg.Flags, msg.Envelope.Date) + continue + } + + fmt.Println("- Server meta-data:") + fmt.Println("UID:", msg.Uid) + fmt.Println("Sequence number:", msg.SeqNum) + fmt.Println("Flags:", msg.Flags) + fmt.Println("Body size:", msg.Size) + fmt.Println("Internal date:", msg.InternalDate.Unix(), msg.InternalDate) + fmt.Println("- Envelope:") + if len(msg.Envelope.From) != 0 { + fmt.Println("From:", FormatAddressList(msg.Envelope.From)) + } + if len(msg.Envelope.To) != 0 { + fmt.Println("To:", FormatAddressList(msg.Envelope.To)) + } + if len(msg.Envelope.Cc) != 0 { + fmt.Println("CC:", FormatAddressList(msg.Envelope.Cc)) + } + if len(msg.Envelope.Bcc) != 0 { + fmt.Println("BCC:", FormatAddressList(msg.Envelope.Bcc)) + } + if msg.Envelope.InReplyTo != "" { + fmt.Println("In-Reply-To:", msg.Envelope.InReplyTo) + } + if msg.Envelope.MessageId != "" { + fmt.Println("Message-Id:", msg.Envelope.MessageId) + } + if !msg.Envelope.Date.IsZero() { + fmt.Println("Date:", msg.Envelope.Date.Unix(), msg.Envelope.Date) + } + if msg.Envelope.Subject != "" { + fmt.Println("Subject:", msg.Envelope.Subject) + } + fmt.Println() + } + return err +} + +func msgsDump(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + mboxName := ctx.Args().Get(1) + if mboxName == "" { + return cli.Exit("Error: MAILBOX is required", 2) + } + seqset := ctx.Args().Get(2) + uid := ctx.Bool("uid") + if seqset == "" { + seqset = "1:*" + uid = true + } else if !uid { + fmt.Fprintln(os.Stderr, "WARNING: --uid=true will be the default in 0.7") + } + + seq, err := imap.ParseSeqSet(seqset) + if err != nil { + return err + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + _, mbox, err := u.GetMailbox(mboxName, true, nil) + if err != nil { + return err + } + + ch := make(chan *imap.Message, 10) + go func() { + err = mbox.ListMessages(uid, seq, []imap.FetchItem{imap.FetchRFC822}, ch) + }() + + for msg := range ch { + for _, v := range msg.Body { + if _, err := io.Copy(os.Stdout, v); err != nil { + return err + } + } + } + return err +} + +func msgsFlags(be module.Storage, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + name := ctx.Args().Get(1) + if name == "" { + return cli.Exit("Error: MAILBOX is required", 2) + } + seqStr := ctx.Args().Get(2) + if seqStr == "" { + return cli.Exit("Error: SEQ is required", 2) + } + + if !ctx.Bool("uid") { + fmt.Fprintln(os.Stderr, "WARNING: --uid=true will be the default in 0.7") + } + + seq, err := imap.ParseSeqSet(seqStr) + if err != nil { + return err + } + + u, err := be.GetIMAPAcct(username) + if err != nil { + return err + } + + _, mbox, err := u.GetMailbox(name, false, nil) + if err != nil { + return err + } + + flags := ctx.Args().Slice()[3:] + if len(flags) == 0 { + return cli.Exit("Error: at least once FLAG is required", 2) + } + + var op imap.FlagsOp + switch ctx.Command.Name { + case "add-flags": + op = imap.AddFlags + case "rem-flags": + op = imap.RemoveFlags + case "set-flags": + op = imap.SetFlags + default: + panic("unknown command: " + ctx.Command.Name) + } + + return mbox.UpdateMessagesFlags(ctx.Bool("uid"), seq, op, true, flags) +} diff --git a/internal/cli/ctl/imapacct.go b/internal/cli/ctl/imapacct.go new file mode 100644 index 0000000..2541a22 --- /dev/null +++ b/internal/cli/ctl/imapacct.go @@ -0,0 +1,301 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package ctl + +import ( + "errors" + "fmt" + "os" + + "github.com/emersion/go-imap" + "github.com/foxcpp/maddy/framework/module" + maddycli "github.com/foxcpp/maddy/internal/cli" + clitools2 "github.com/foxcpp/maddy/internal/cli/clitools" + "github.com/urfave/cli/v2" +) + +func init() { + maddycli.AddSubcommand( + &cli.Command{ + Name: "imap-acct", + Usage: "IMAP storage accounts management", + Description: `These subcommands can be used to list/create/delete IMAP storage +accounts for any storage backend supported by maddy. + +The corresponding storage backend should be configured in maddy.conf and be +defined in a top-level configuration block. By default, the name of that +block should be local_mailboxes but this can be changed using --cfg-block +flag for subcommands. + +Note that in default configuration it is not enough to create an IMAP storage +account to grant server access. Additionally, user credentials should +be created using 'creds' subcommand. +`, + Subcommands: []*cli.Command{ + { + Name: "list", + Usage: "List storage accounts", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return imapAcctList(be, ctx) + }, + }, + { + Name: "create", + Usage: "Create IMAP storage account", + Description: `In addition to account creation, this command +creates a set of default folder (mailboxes) with special-use attribute set.`, + ArgsUsage: "USERNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "no-specialuse", + Usage: "Do not create special-use folders", + Value: false, + }, + &cli.StringFlag{ + Name: "sent-name", + Usage: "Name of special mailbox for sent messages, use empty string to not create any", + Value: "Sent", + }, + &cli.StringFlag{ + Name: "trash-name", + Usage: "Name of special mailbox for trash, use empty string to not create any", + Value: "Trash", + }, + &cli.StringFlag{ + Name: "junk-name", + Usage: "Name of special mailbox for 'junk' (spam), use empty string to not create any", + Value: "Junk", + }, + &cli.StringFlag{ + Name: "drafts-name", + Usage: "Name of special mailbox for drafts, use empty string to not create any", + Value: "Drafts", + }, + &cli.StringFlag{ + Name: "archive-name", + Usage: "Name of special mailbox for archive, use empty string to not create any", + Value: "Archive", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return imapAcctCreate(be, ctx) + }, + }, + { + Name: "remove", + Usage: "Delete IMAP storage account", + Description: `If IMAP connections are open and using the specified account, +messages access will be killed off immediately though connection will remain open. No cache +or other buffering takes effect.`, + ArgsUsage: "USERNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.BoolFlag{ + Name: "yes", + Aliases: []string{"y"}, + Usage: "Don't ask for confirmation", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return imapAcctRemove(be, ctx) + }, + }, + { + Name: "appendlimit", + Usage: "Query or set accounts's APPENDLIMIT value", + Description: `APPENDLIMIT value determines the size of a message that +can be saved into a mailbox using IMAP APPEND command. This does not affect the size +of messages that can be delivered to the mailbox from non-IMAP sources (e.g. SMTP). + +Global APPENDLIMIT value set via server configuration takes precedence over +per-account values configured using this command. + +APPENDLIMIT value (either global or per-account) cannot be larger than +4 GiB due to IMAP protocol limitations. +`, + ArgsUsage: "USERNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_mailboxes", + }, + &cli.IntFlag{ + Name: "value", + Aliases: []string{"v"}, + Usage: "Set APPENDLIMIT to specified value (in bytes)", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openStorage(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return imapAcctAppendlimit(be, ctx) + }, + }, + }, + }) +} + +type SpecialUseUser interface { + CreateMailboxSpecial(name, specialUseAttr string) error +} + +func imapAcctList(be module.Storage, ctx *cli.Context) error { + mbe, ok := be.(module.ManageableStorage) + if !ok { + return cli.Exit("Error: storage backend does not support accounts management using maddy command", 2) + } + + list, err := mbe.ListIMAPAccts() + if err != nil { + return err + } + + if len(list) == 0 && !ctx.Bool("quiet") { + fmt.Fprintln(os.Stderr, "No users.") + } + + for _, user := range list { + fmt.Println(user) + } + return nil +} + +func imapAcctCreate(be module.Storage, ctx *cli.Context) error { + mbe, ok := be.(module.ManageableStorage) + if !ok { + return cli.Exit("Error: storage backend does not support accounts management using maddy command", 2) + } + + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + + if err := mbe.CreateIMAPAcct(username); err != nil { + return err + } + + act, err := mbe.GetIMAPAcct(username) + if err != nil { + return fmt.Errorf("failed to get user: %w", err) + } + + suu, ok := act.(SpecialUseUser) + if !ok { + fmt.Fprintf(os.Stderr, "Note: Storage backend does not support SPECIAL-USE IMAP extension") + } + + if ctx.Bool("no-specialuse") { + return nil + } + + createMbox := func(name, specialUseAttr string) error { + if suu == nil { + return act.CreateMailbox(name) + } + return suu.CreateMailboxSpecial(name, specialUseAttr) + } + + if name := ctx.String("sent-name"); name != "" { + if err := createMbox(name, imap.SentAttr); err != nil { + fmt.Fprintf(os.Stderr, "Failed to create sent folder: %v", err) + } + } + if name := ctx.String("trash-name"); name != "" { + if err := createMbox(name, imap.TrashAttr); err != nil { + fmt.Fprintf(os.Stderr, "Failed to create trash folder: %v", err) + } + } + if name := ctx.String("junk-name"); name != "" { + if err := createMbox(name, imap.JunkAttr); err != nil { + fmt.Fprintf(os.Stderr, "Failed to create junk folder: %v", err) + } + } + if name := ctx.String("drafts-name"); name != "" { + if err := createMbox(name, imap.DraftsAttr); err != nil { + fmt.Fprintf(os.Stderr, "Failed to create drafts folder: %v", err) + } + } + if name := ctx.String("archive-name"); name != "" { + if err := createMbox(name, imap.ArchiveAttr); err != nil { + fmt.Fprintf(os.Stderr, "Failed to create archive folder: %v", err) + } + } + + return nil +} + +func imapAcctRemove(be module.Storage, ctx *cli.Context) error { + mbe, ok := be.(module.ManageableStorage) + if !ok { + return cli.Exit("Error: storage backend does not support accounts management using maddy command", 2) + } + + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + + if !ctx.Bool("yes") { + if !clitools2.Confirmation("Are you sure you want to delete this user account?", false) { + return errors.New("Cancelled") + } + } + + return mbe.DeleteIMAPAcct(username) +} diff --git a/internal/cli/ctl/moduleinit.go b/internal/cli/ctl/moduleinit.go new file mode 100644 index 0000000..23e79e5 --- /dev/null +++ b/internal/cli/ctl/moduleinit.go @@ -0,0 +1,133 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package ctl + +import ( + "errors" + "fmt" + "io" + "os" + + "github.com/foxcpp/maddy" + parser "github.com/foxcpp/maddy/framework/cfgparser" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/updatepipe" + "github.com/urfave/cli/v2" +) + +func closeIfNeeded(i interface{}) { + if c, ok := i.(io.Closer); ok { + c.Close() + } +} + +func getCfgBlockModule(ctx *cli.Context) (map[string]interface{}, *maddy.ModInfo, error) { + cfgPath := ctx.String("config") + if cfgPath == "" { + return nil, nil, cli.Exit("Error: config is required", 2) + } + cfgFile, err := os.Open(cfgPath) + if err != nil { + return nil, nil, cli.Exit(fmt.Sprintf("Error: failed to open config: %v", err), 2) + } + defer cfgFile.Close() + cfgNodes, err := parser.Read(cfgFile, cfgFile.Name()) + if err != nil { + return nil, nil, cli.Exit(fmt.Sprintf("Error: failed to parse config: %v", err), 2) + } + + globals, cfgNodes, err := maddy.ReadGlobals(cfgNodes) + if err != nil { + return nil, nil, err + } + + if err := maddy.InitDirs(); err != nil { + return nil, nil, err + } + + module.NoRun = true + _, mods, err := maddy.RegisterModules(globals, cfgNodes) + if err != nil { + return nil, nil, err + } + defer hooks.RunHooks(hooks.EventShutdown) + + cfgBlock := ctx.String("cfg-block") + if cfgBlock == "" { + return nil, nil, cli.Exit("Error: cfg-block is required", 2) + } + var mod maddy.ModInfo + for _, m := range mods { + if m.Instance.InstanceName() == cfgBlock { + mod = m + break + } + } + if mod.Instance == nil { + return nil, nil, cli.Exit(fmt.Sprintf("Error: unknown configuration block: %s", cfgBlock), 2) + } + + return globals, &mod, nil +} + +func openStorage(ctx *cli.Context) (module.Storage, error) { + globals, mod, err := getCfgBlockModule(ctx) + if err != nil { + return nil, err + } + + storage, ok := mod.Instance.(module.Storage) + if !ok { + return nil, cli.Exit(fmt.Sprintf("Error: configuration block %s is not an IMAP storage", ctx.String("cfg-block")), 2) + } + + if err := mod.Instance.Init(config.NewMap(globals, mod.Cfg)); err != nil { + return nil, fmt.Errorf("Error: module initialization failed: %w", err) + } + + if updStore, ok := mod.Instance.(updatepipe.Backend); ok { + if err := updStore.EnableUpdatePipe(updatepipe.ModePush); err != nil && !errors.Is(err, os.ErrNotExist) { + fmt.Fprintf(os.Stderr, "Failed to initialize update pipe, do not remove messages from mailboxes open by clients: %v\n", err) + } + } else { + fmt.Fprintf(os.Stderr, "No update pipe support, do not remove messages from mailboxes open by clients\n") + } + + return storage, nil +} + +func openUserDB(ctx *cli.Context) (module.PlainUserDB, error) { + globals, mod, err := getCfgBlockModule(ctx) + if err != nil { + return nil, err + } + + userDB, ok := mod.Instance.(module.PlainUserDB) + if !ok { + return nil, cli.Exit(fmt.Sprintf("Error: configuration block %s is not a local credentials store", ctx.String("cfg-block")), 2) + } + + if err := mod.Instance.Init(config.NewMap(globals, mod.Cfg)); err != nil { + return nil, fmt.Errorf("Error: module initialization failed: %w", err) + } + + return userDB, nil +} diff --git a/internal/cli/ctl/users.go b/internal/cli/ctl/users.go new file mode 100644 index 0000000..09dc909 --- /dev/null +++ b/internal/cli/ctl/users.go @@ -0,0 +1,246 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package ctl + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth/pass_table" + maddycli "github.com/foxcpp/maddy/internal/cli" + clitools2 "github.com/foxcpp/maddy/internal/cli/clitools" + "github.com/urfave/cli/v2" + "golang.org/x/crypto/bcrypt" +) + +func init() { + maddycli.AddSubcommand( + &cli.Command{ + Name: "creds", + Usage: "Local credentials management", + Description: `These commands manipulate credential databases used by +maddy mail server. + +Corresponding credential database should be defined in maddy.conf as +a top-level config block. By default the block name should be local_authdb ( +can be changed using --cfg-block argument for subcommands). + +Note that it is not enough to create user credentials in order to grant +IMAP access - IMAP account should be also created using 'imap-acct create' subcommand. +`, + Subcommands: []*cli.Command{ + { + Name: "list", + Usage: "List created credentials", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_authdb", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openUserDB(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return usersList(be, ctx) + }, + }, + { + Name: "create", + Usage: "Create user account", + Description: `Reads password from stdin. + +If configuration block uses auth.pass_table, then hash algorithm can be configured +using command flags. Otherwise, these options cannot be used. +`, + ArgsUsage: "USERNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_authdb", + }, + &cli.StringFlag{ + Name: "password", + Aliases: []string{"p"}, + Usage: "Use `PASSWORD instead of reading password from stdin.\n\t\tWARNING: Provided only for debugging convenience. Don't leave your passwords in shell history!", + }, + &cli.StringFlag{ + Name: "hash", + Usage: "Use specified hash algorithm. Valid values: " + strings.Join(pass_table.Hashes, ", "), + Value: "bcrypt", + }, + &cli.IntFlag{ + Name: "bcrypt-cost", + Usage: "Specify bcrypt cost value", + Value: bcrypt.DefaultCost, + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openUserDB(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return usersCreate(be, ctx) + }, + }, + { + Name: "remove", + Usage: "Delete user account", + ArgsUsage: "USERNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_authdb", + }, + &cli.BoolFlag{ + Name: "yes", + Aliases: []string{"y"}, + Usage: "Don't ask for confirmation", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openUserDB(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return usersRemove(be, ctx) + }, + }, + { + Name: "password", + Usage: "Change account password", + Description: "Reads password from stdin", + ArgsUsage: "USERNAME", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cfg-block", + Usage: "Module configuration block to use", + EnvVars: []string{"MADDY_CFGBLOCK"}, + Value: "local_authdb", + }, + &cli.StringFlag{ + Name: "password", + Aliases: []string{"p"}, + Usage: "Use `PASSWORD` instead of reading password from stdin.\n\t\tWARNING: Provided only for debugging convenience. Don't leave your passwords in shell history!", + }, + }, + Action: func(ctx *cli.Context) error { + be, err := openUserDB(ctx) + if err != nil { + return err + } + defer closeIfNeeded(be) + return usersPassword(be, ctx) + }, + }, + }, + }) +} + +func usersList(be module.PlainUserDB, ctx *cli.Context) error { + list, err := be.ListUsers() + if err != nil { + return err + } + + if len(list) == 0 && !ctx.Bool("quiet") { + fmt.Fprintln(os.Stderr, "No users.") + } + + for _, user := range list { + fmt.Println(user) + } + return nil +} + +func usersCreate(be module.PlainUserDB, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return cli.Exit("Error: USERNAME is required", 2) + } + + var pass string + if ctx.IsSet("password") { + pass = ctx.String("password") + } else { + var err error + pass, err = clitools2.ReadPassword("Enter password for new user") + if err != nil { + return err + } + } + + if beHash, ok := be.(*pass_table.Auth); ok { + return beHash.CreateUserHash(username, pass, ctx.String("hash"), pass_table.HashOpts{ + BcryptCost: ctx.Int("bcrypt-cost"), + }) + } else if ctx.IsSet("hash") || ctx.IsSet("bcrypt-cost") { + return cli.Exit("Error: --hash cannot be used with non-pass_table credentials DB", 2) + } else { + return be.CreateUser(username, pass) + } +} + +func usersRemove(be module.PlainUserDB, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return errors.New("Error: USERNAME is required") + } + + if !ctx.Bool("yes") { + if !clitools2.Confirmation("Are you sure you want to delete this user account?", false) { + return errors.New("Cancelled") + } + } + + return be.DeleteUser(username) +} + +func usersPassword(be module.PlainUserDB, ctx *cli.Context) error { + username := ctx.Args().First() + if username == "" { + return errors.New("Error: USERNAME is required") + } + + var pass string + if ctx.IsSet("password") { + pass = ctx.String("password") + } else { + var err error + pass, err = clitools2.ReadPassword("Enter new password") + if err != nil { + return err + } + } + + return be.SetUserPassword(username, pass) +} diff --git a/internal/cli/extflag.go b/internal/cli/extflag.go new file mode 100644 index 0000000..8cfc27c --- /dev/null +++ b/internal/cli/extflag.go @@ -0,0 +1,60 @@ +package maddycli + +import ( + "flag" + + "github.com/urfave/cli/v2" +) + +// extFlag implements cli.Flag via standard flag.Flag. +type extFlag struct { + f *flag.Flag +} + +func (e *extFlag) Apply(fs *flag.FlagSet) error { + fs.Var(e.f.Value, e.f.Name, e.f.Usage) + return nil +} + +func (e *extFlag) Names() []string { + return []string{e.f.Name} +} + +func (e *extFlag) IsSet() bool { + return false +} + +func (e *extFlag) String() string { + return cli.FlagStringer(e) +} + +func (e *extFlag) IsVisible() bool { + return true +} + +func (e *extFlag) TakesValue() bool { + return false +} + +func (e *extFlag) GetUsage() string { + return e.f.Usage +} + +func (e *extFlag) GetValue() string { + return e.f.Value.String() +} + +func (e *extFlag) GetDefaultText() string { + return e.f.DefValue +} + +func (e *extFlag) GetEnvVars() []string { + return nil +} + +func mapStdlibFlags(app *cli.App) { + // Modified AllowExtFlags from cli lib with -test.* exception removed. + flag.VisitAll(func(f *flag.Flag) { + app.Flags = append(app.Flags, &extFlag{f}) + }) +} diff --git a/internal/dmarc/dmarc.go b/internal/dmarc/dmarc.go new file mode 100644 index 0000000..723fedd --- /dev/null +++ b/internal/dmarc/dmarc.go @@ -0,0 +1,42 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dmarc + +import ( + "context" + + "github.com/emersion/go-msgauth/dmarc" +) + +type ( + Resolver interface { + LookupTXT(context.Context, string) ([]string, error) + } + + Record = dmarc.Record + Policy = dmarc.Policy + AlignmentMode = dmarc.AlignmentMode + FailureOptions = dmarc.FailureOptions +) + +const ( + PolicyNone = dmarc.PolicyNone + PolicyReject = dmarc.PolicyReject + PolicyQuarantine = dmarc.PolicyQuarantine +) diff --git a/internal/dmarc/evaluate.go b/internal/dmarc/evaluate.go new file mode 100644 index 0000000..ff978e5 --- /dev/null +++ b/internal/dmarc/evaluate.go @@ -0,0 +1,256 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dmarc + +import ( + "context" + "errors" + "fmt" + "net" + "net/mail" + "strings" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/emersion/go-msgauth/dmarc" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/dns" + "golang.org/x/net/publicsuffix" +) + +// FetchRecord looks up the DMARC record relevant for the RFC5322.From domain. +// It returns the record and the domain it was found with (may not be +// equal to the RFC5322.From domain). +func FetchRecord(ctx context.Context, r Resolver, fromDomain string) (policyDomain string, rec *Record, err error) { + policyDomain = fromDomain + + // 1. Lookup using From Domain. + txts, err := r.LookupTXT(ctx, dns.FQDN("_dmarc."+fromDomain)) + if err != nil { + dnsErr, ok := err.(*net.DNSError) + if !ok || !dnsErr.IsNotFound { + return "", nil, err + } + } + if len(txts) == 0 { + // No records or 'no such host', try orgDomain. + orgDomain, err := publicsuffix.EffectiveTLDPlusOne(fromDomain) + if err != nil { + return "", nil, err + } + + policyDomain = orgDomain + + txts, err = r.LookupTXT(ctx, dns.FQDN("_dmarc."+orgDomain)) + if err != nil { + dnsErr, ok := err.(*net.DNSError) + if !ok || !dnsErr.IsNotFound { + return "", nil, err + } + } + // Still nothing? Bail out. + if len(txts) == 0 { + return "", nil, nil + } + } + + // Exclude records that are not DMARC policies. + records := txts[:0] + for _, txt := range txts { + if strings.HasPrefix(txt, "v=DMARC1") { + records = append(records, txt) + } + } + // Multiple records => no record. + if len(records) > 1 || len(records) == 0 { + return "", nil, nil + } + + rec, err = dmarc.Parse(records[0]) + + return policyDomain, rec, err +} + +type EvalResult struct { + // The Authentication-Results field generated as a result of the DMARC + // check. + Authres authres.DMARCResult + + // The Authentication-Results field for SPF that was considered during + // alignment check. May be empty. + SPFResult authres.SPFResult + + // Whether HELO or MAIL FROM match the RFC5322.From domain. + SPFAligned bool + + // The Authentication-Results field for the DKIM signature that is aligned, + // if no signatures are aligned - this field contains the result for the + // first signature. May be empty. + DKIMResult authres.DKIMResult + + // Whether there is a DKIM signature with the d= field matching the + // RFC5322.From domain. + DKIMAligned bool +} + +// EvaluateAlignment checks whether identifiers authenticated by SPF and DKIM are in alignment +// with the RFC5322.Domain. +// +// It returns EvalResult which contains the Authres field with the actual check result and +// a bunch of other trace information that can be useful for troubleshooting +// (and also report generation). +func EvaluateAlignment(fromDomain string, record *Record, results []authres.Result) EvalResult { + var ( + spfAligned = false + spfResult = authres.SPFResult{} + dkimAligned = false + dkimResult = authres.DKIMResult{} + dkimPresent = false + dkimTempFail = false + ) + for _, res := range results { + if dkimRes, ok := res.(*authres.DKIMResult); ok { + dkimPresent = true + + // We want to return DKIM result for a signature provided by the orgDomain, + // in case there is none - return any (possibly misaligned) for reference. + if dkimResult.Value == "" { + dkimResult = *dkimRes + } + if isAligned(fromDomain, dkimRes.Domain, record.DKIMAlignment) { + dkimResult = *dkimRes + switch dkimRes.Value { + case authres.ResultPass: + dkimAligned = true + case authres.ResultTempError: + dkimTempFail = true + } + } + } + if spfRes, ok := res.(*authres.SPFResult); ok { + spfResult = *spfRes + var aligned bool + if spfRes.From == "" { + aligned = isAligned(fromDomain, spfRes.Helo, record.SPFAlignment) + } else { + aligned = isAligned(fromDomain, spfRes.From, record.SPFAlignment) + } + if aligned && spfRes.Value == authres.ResultPass { + spfAligned = true + } + } + } + + res := EvalResult{ + SPFResult: spfResult, + SPFAligned: spfAligned, + DKIMResult: dkimResult, + DKIMAligned: dkimAligned, + } + + if !dkimPresent || spfResult.Value == "" { + res.Authres = authres.DMARCResult{ + Value: authres.ResultNone, + Reason: "Not enough information (required checks are disabled)", + From: fromDomain, + } + return res + } + + if dkimTempFail && !dkimAligned && !spfAligned { + // We can't be sure whether it is aligned or not. Bail out. + res.Authres = authres.DMARCResult{ + Value: authres.ResultTempError, + Reason: "DKIM authentication temp error", + From: fromDomain, + } + return res + } + if !dkimAligned && spfResult.Value == authres.ResultTempError { + // We can't be sure whether it is aligned or not. Bail out. + res.Authres = authres.DMARCResult{ + Value: authres.ResultTempError, + Reason: "SPF authentication temp error", + From: fromDomain, + } + return res + } + + res.Authres.From = fromDomain + if dkimAligned || spfAligned { + res.Authres.Value = authres.ResultPass + } else { + res.Authres.Value = authres.ResultFail + res.Authres.Reason = "No aligned identifiers" + } + return res +} + +func isAligned(fromDomain, authDomain string, mode AlignmentMode) bool { + if mode == dmarc.AlignmentStrict { + return strings.EqualFold(fromDomain, authDomain) + } + + tld, _ := publicsuffix.PublicSuffix(fromDomain) + if strings.EqualFold(fromDomain, tld) { + return strings.EqualFold(fromDomain, authDomain) + } + orgDomainFrom, err := publicsuffix.EffectiveTLDPlusOne(fromDomain) + if err != nil { + return false + } + authDomainFrom, err := publicsuffix.EffectiveTLDPlusOne(authDomain) + if err != nil { + return false + } + + return strings.EqualFold(orgDomainFrom, authDomainFrom) +} + +func ExtractFromDomain(hdr textproto.Header) (string, error) { + // TODO(GH emersion/go-message#75): Add textproto.Header.Count method. + var firstFrom string + for fields := hdr.FieldsByKey("From"); fields.Next(); { + if firstFrom == "" { + firstFrom = fields.Value() + } else { + return "", errors.New("dmarc: multiple From header fields are not allowed") + } + } + if firstFrom == "" { + return "", errors.New("dmarc: missing From header field") + } + + hdrFromList, err := mail.ParseAddressList(firstFrom) + if err != nil { + return "", fmt.Errorf("dmarc: malformed From header field: %s", strings.TrimPrefix(err.Error(), "mail: ")) + } + if len(hdrFromList) > 1 { + return "", errors.New("dmarc: multiple addresses in From field are not allowed") + } + if len(hdrFromList) == 0 { + return "", errors.New("dmarc: missing address in From field") + } + _, domain, err := address.Split(hdrFromList[0].Address) + if err != nil { + return "", fmt.Errorf("dmarc: malformed From header field: %w", err) + } + + return domain, nil +} diff --git a/internal/dmarc/evaluate_test.go b/internal/dmarc/evaluate_test.go new file mode 100644 index 0000000..c44bbbe --- /dev/null +++ b/internal/dmarc/evaluate_test.go @@ -0,0 +1,514 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dmarc + +import ( + "bufio" + "strings" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/emersion/go-msgauth/dmarc" +) + +func TestEvaluateAlignment(t *testing.T) { + type tCase struct { + fromDomain string + record *Record + results []authres.Result + + output authres.ResultValue + } + test := func(i int, c tCase) { + out := EvaluateAlignment(c.fromDomain, c.record, c.results) + t.Logf("%d - %+v", i, out) + if out.Authres.Value != c.output { + t.Errorf("%d: Wrong eval result, want '%s', got '%s' (%+v)", i, c.output, out.Authres.Value, out) + } + } + + cases := []tCase{ + { // 0 + fromDomain: "example.org", + record: &Record{}, + + output: authres.ResultNone, + }, + { // 1 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultFail, + From: "example.org", + Helo: "mx.example.org", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultFail, + }, + { // 2 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "example.org", + Helo: "mx.example.org", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultPass, + }, + { // 3 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultFail, + From: "example.org", + Helo: "mx.example.org", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultFail, + }, + { // 4 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "example.com", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultFail, + }, + { // 5 + fromDomain: "example.com", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "cbg.bounces.example.com", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultPass, + }, + { // 6 + fromDomain: "example.com", + record: &Record{ + SPFAlignment: dmarc.AlignmentStrict, + }, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "cbg.bounces.example.com", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultFail, + }, + { // 7 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultFail, + Domain: "example.org", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultFail, + }, + { // 8 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "example.org", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultPass, + }, + { // 9 + fromDomain: "example.com", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "cbg.bounces.example.com", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "example.com", + }, + }, + output: authres.ResultPass, + }, + { // 10 + fromDomain: "example.com", + record: &Record{ + SPFAlignment: dmarc.AlignmentRelaxed, + }, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "cbg.bounces.example.com", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultFail, + Domain: "example.com", + }, + }, + output: authres.ResultPass, + }, + { // 11 + fromDomain: "example.com", + record: &Record{ + SPFAlignment: dmarc.AlignmentStrict, + }, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "cbg.bounces.example.com", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "example.com", + }, + }, + output: authres.ResultPass, + }, + { // 12 + fromDomain: "example.com", + record: &Record{ + SPFAlignment: dmarc.AlignmentStrict, + DKIMAlignment: dmarc.AlignmentStrict, + }, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "cbg.bounces.example.com", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultFail, + Domain: "cbg.example.com", + }, + }, + output: authres.ResultFail, + }, + { // 13 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultFail, + Domain: "example.org", + }, + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "example.net", + }, + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "example.org", + }, + &authres.DKIMResult{ + Value: authres.ResultFail, + Domain: "example.com", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultPass, + }, + { // 14 + fromDomain: "example.com", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultPass, + }, + { // 15 + fromDomain: "example.com", + record: &Record{ + SPFAlignment: dmarc.AlignmentStrict, + }, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultFail, + }, + { // 16 + fromDomain: "example.com", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultTempError, + From: "", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + }, + output: authres.ResultTempError, + }, + { // 17 + fromDomain: "example.com", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultTempError, + Domain: "example.com", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultTempError, + }, + { // 18 + fromDomain: "example.com", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultTempError, + From: "", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "example.com", + }, + }, + output: authres.ResultPass, + }, + { // 19 + fromDomain: "example.com", + record: &Record{}, + results: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultPass, + From: "", + Helo: "mx.example.com", + }, + &authres.DKIMResult{ + Value: authres.ResultTempError, + Domain: "example.com", + }, + }, + output: authres.ResultPass, + }, + { // 20 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "example.org", + }, + &authres.DKIMResult{ + Value: authres.ResultTempError, + Domain: "example.org", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultPass, + }, + { // 21 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultFail, + Domain: "example.org", + }, + &authres.DKIMResult{ + Value: authres.ResultTempError, + Domain: "example.org", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultTempError, + }, + { // 22 + fromDomain: "example.org", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultNone, + Domain: "example.org", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultFail, + }, + { // 23 + fromDomain: "sub.example.org", + record: &Record{}, + results: []authres.Result{ + &authres.DKIMResult{ + Value: authres.ResultPass, + Domain: "mx.example.org", + }, + &authres.SPFResult{ + Value: authres.ResultNone, + From: "example.org", + Helo: "mx.example.org", + }, + }, + output: authres.ResultPass, + }, + } + for i, case_ := range cases { + test(i, case_) + } +} + +func TestExtractDomains(t *testing.T) { + type tCase struct { + hdr string + + fromDomain string + } + test := func(i int, c tCase) { + hdr, err := textproto.ReadHeader(bufio.NewReader(strings.NewReader(c.hdr + "\n\n"))) + if err != nil { + panic(err) + } + + domain, err := ExtractFromDomain(hdr) + if c.fromDomain == "" && err == nil { + t.Errorf("%d: expected failure, got fromDomain = %s", i, domain) + return + } + if c.fromDomain != "" && err != nil { + t.Errorf("%d: unexpected error: %v", i, err) + return + } + if domain != c.fromDomain { + t.Errorf("%d: want fromDomain = %v but got %s", i, c.fromDomain, domain) + } + } + + cases := []tCase{ + { + hdr: `From: `, + fromDomain: "example.org", + }, + { + hdr: `From: `, + fromDomain: "foo.example.org", + }, + { + hdr: `From: , `, + }, + { + hdr: `From: , +From: `, + }, + { + hdr: `From: `, + }, + { + hdr: `From: `, + }, + { + hdr: `From: foo`, + }, + } + for i, case_ := range cases { + test(i, case_) + } +} diff --git a/internal/dmarc/verifier.go b/internal/dmarc/verifier.go new file mode 100644 index 0000000..a79b526 --- /dev/null +++ b/internal/dmarc/verifier.go @@ -0,0 +1,174 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dmarc + +import ( + "context" + "math/rand" + "net" + "runtime/trace" + "strings" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/emersion/go-msgauth/dmarc" +) + +type verifyData struct { + policyDomain string + fromDomain string + record *Record + recordErr error +} + +// errPanic is used to propagate the panic() from the FetchRecord +// goroutine to the goroutine that called Apply. +type errPanic struct { + err interface{} +} + +func (errPanic) Error() string { + return "panic during policy fetch" +} + +// Verifier is the structure that wraps all state necessary to verify a +// single message using DMARC checks. +// +// It cannot be reused. +type Verifier struct { + fetchCh chan verifyData + fetchCancel context.CancelFunc + + resolver Resolver + + // TODO(GH #206): DMARC reporting + // FailureReportFunc is the callback that is called when a failure report + // is generated. If it is nil - failure reports generation is disabled. + // FailureReportFunc func(textproto.Header, io.Reader) +} + +func NewVerifier(r Resolver) *Verifier { + return &Verifier{ + fetchCh: make(chan verifyData, 1), + resolver: r, + } +} + +func (v *Verifier) Close() error { + if v.fetchCancel != nil { + v.fetchCancel() + } + return nil +} + +// FetchRecord prepares the Verifier by starting the policy lookup. Lookup is +// performed asynchronously to improve performance. +// +// If panic occurs in the lookup goroutine - call to Apply will panic. +func (v *Verifier) FetchRecord(ctx context.Context, header textproto.Header) { + fromDomain, err := ExtractFromDomain(header) + if err != nil { + v.fetchCh <- verifyData{ + recordErr: err, + } + return + } + + ctx, v.fetchCancel = context.WithCancel(ctx) + go func() { + defer func() { + if err := recover(); err != nil { + v.fetchCh <- verifyData{ + recordErr: errPanic{err: err}, + } + } + }() + + defer trace.StartRegion(ctx, "DMARC/FetchRecord").End() + + policyDomain, record, err := FetchRecord(ctx, v.resolver, fromDomain) + v.fetchCh <- verifyData{ + policyDomain: policyDomain, + fromDomain: fromDomain, + record: record, + recordErr: err, + } + }() +} + +// Apply actually performs all actions necessary to apply a DMARC policy to the message. +// +// The authRes slice should contain results for DKIM and SPF checks. FetchRecord should be +// caled before calling this function. +// +// It returns the Authentication-Result field to be included in the message (as +// a part of the EvalResult struct) and the appropriate action that should be +// taken by the MTA. In case of PolicyReject, caller should inspect the +// Result.Value to determine whether to use a temporary or permanent error code +// as Apply implements the 'fail closed' strategy for handling of temporary +// errors. +// +// Additionally, it relies on the math/rand default source to be initialized to determine +// whether to apply a policy with the pct key. +func (v *Verifier) Apply(authRes []authres.Result) (EvalResult, Policy) { + data := <-v.fetchCh + if data.recordErr != nil { + result := authres.DMARCResult{ + Value: authres.ResultPermError, + Reason: "Policy lookup failed: " + data.recordErr.Error(), + // If may be empty, but it is fine (it will not be included in the field then). + From: data.fromDomain, + } + if dnsErr, ok := data.recordErr.(*net.DNSError); ok && dnsErr.Temporary() { + result.Value = authres.ResultTempError + // 'fail closed' behavior, reject the message if a temporary error + // occurs. + return EvalResult{ + Authres: result, + }, dmarc.PolicyReject + } + return EvalResult{ + Authres: result, + }, dmarc.PolicyNone + } + if data.record == nil { + return EvalResult{ + Authres: authres.DMARCResult{ + Value: authres.ResultNone, + From: data.fromDomain, + }, + }, dmarc.PolicyNone + } + + result := EvaluateAlignment(data.fromDomain, data.record, authRes) + if result.Authres.Value == authres.ResultPass || result.Authres.Value == authres.ResultNone { + return result, dmarc.PolicyNone + } + + if data.record.Percent != nil && rand.Int31n(100) > int32(*data.record.Percent) { + return result, dmarc.PolicyNone + } + + policy := data.record.Policy + if !strings.EqualFold(data.policyDomain, data.fromDomain) && data.record.SubdomainPolicy != "" { + policy = data.record.SubdomainPolicy + } + + return result, policy +} diff --git a/internal/dmarc/verifier_test.go b/internal/dmarc/verifier_test.go new file mode 100644 index 0000000..1adfe99 --- /dev/null +++ b/internal/dmarc/verifier_test.go @@ -0,0 +1,219 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dmarc + +import ( + "bufio" + "context" + "errors" + "net" + "strings" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/foxcpp/go-mockdns" +) + +func TestDMARC(t *testing.T) { + test := func(zones map[string]mockdns.Zone, hdr string, authres []authres.Result, policyApplied Policy, dmarcRes authres.ResultValue) { + t.Helper() + v := NewVerifier(&mockdns.Resolver{Zones: zones}) + defer v.Close() + + hdrParsed, err := textproto.ReadHeader(bufio.NewReader(strings.NewReader(hdr))) + if err != nil { + panic(err) + } + v.FetchRecord(context.Background(), hdrParsed) + evalRes, policy := v.Apply(authres) + + if policy != policyApplied { + t.Errorf("expected applied policy to be '%v', got '%v'", policyApplied, policy) + } + if evalRes.Authres.Value != dmarcRes { + t.Errorf("expected DMARC result to be '%v', got '%v'", dmarcRes, evalRes.Authres.Value) + } + } + + // No policy => DMARC 'none' + test(map[string]mockdns.Zone{}, "From: hello@example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultNone) + + // Policy present & identifiers align => DMARC 'pass' + test(map[string]mockdns.Zone{ + "_dmarc.example.org.": { + TXT: []string{"v=DMARC1; p=none"}, + }, + }, "From: hello@example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultPass) + + // No SPF check run => DMARC 'none', no action taken + test(map[string]mockdns.Zone{ + "_dmarc.example.org.": { + TXT: []string{"v=DMARC1; p=reject"}, + }, + }, "From: hello@example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + }, PolicyNone, authres.ResultNone) + + // No DKIM check run => DMARC 'none', no action taken + test(map[string]mockdns.Zone{ + "_dmarc.example.org.": { + TXT: []string{"v=DMARC1; p=reject"}, + }, + }, "From: hello@example.org\r\n\r\n", []authres.Result{ + &authres.SPFResult{Value: authres.ResultPass, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultNone) + + // Check org. domain and from domain, prefer from domain. + // https://tools.ietf.org/html/rfc7489#section-6.6.3 + test(map[string]mockdns.Zone{ + "_dmarc.example.org.": { + TXT: []string{"v=DMARC1; p=none"}, + }, + }, "From: hello@sub.example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultPass) + test(map[string]mockdns.Zone{ + "_dmarc.sub.example.org.": { + TXT: []string{"v=DMARC1; p=none"}, + }, + }, "From: hello@sub.example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultPass) + test(map[string]mockdns.Zone{ + "_dmarc.sub.example.org.": { + TXT: []string{"v=DMARC1; p=none"}, + }, + "_dmarc.example.org.": { + TXT: []string{"v=malformed"}, + }, + }, "From: hello@sub.example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultPass) + + // Non-DMARC records are ignored. + // https://tools.ietf.org/html/rfc7489#section-6.6.3 + test(map[string]mockdns.Zone{ + "_dmarc.example.org.": { + TXT: []string{"ignore", "v=DMARC1; p=none"}, + }, + }, "From: hello@sub.example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultPass) + + // Multiple policies => no policy. + // https://tools.ietf.org/html/rfc7489#section-6.6.3 + test(map[string]mockdns.Zone{ + "_dmarc.example.org.": { + TXT: []string{"v=DMARC1; p=reject", "v=DMARC1; p=none"}, + }, + }, "From: hello@sub.example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultNone) + + // Malformed policy => no policy + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + TXT: []string{"v=aaaa"}, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultNone) + + // Policy fetch error => DMARC 'permerror' but the message + // is accepted. + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + Err: errors.New("the dns server is going insane"), + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultPermError) + + // Policy fetch error => DMARC 'temperror' but the message + // is accepted ("fail closed") + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + Err: &net.DNSError{ + Err: "the dns server is going insane, temporary", + IsTemporary: true, + }, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyReject, authres.ResultTempError) + + // Misaligned From vs DKIM => DMARC 'fail'. + // Side note: More comprehensive tests for alignment evaluation + // can be found in check/dmarc/evaluate_test.go. This test merely checks + // that the correct action is taken based on the policy. + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + TXT: []string{"v=DMARC1; p=none"}, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultFail) + + // Misaligned From vs DKIM => DMARC 'fail', policy says to reject + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + TXT: []string{"v=DMARC1; p=reject"}, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyReject, authres.ResultFail) + + // Misaligned From vs DKIM => DMARC 'fail' + // Subdomain policy requests no action, main domain policy says to reject. + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + TXT: []string{"v=DMARC1; sp=none; p=reject"}, + }, + }, "From: hello@sub.example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyNone, authres.ResultFail) + + // Misaligned From vs DKIM => DMARC 'fail', policy says to quarantine. + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + TXT: []string{"v=DMARC1; p=quarantine"}, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, PolicyQuarantine, authres.ResultFail) +} diff --git a/internal/dsn/dsn.go b/internal/dsn/dsn.go new file mode 100644 index 0000000..59707a7 --- /dev/null +++ b/internal/dsn/dsn.go @@ -0,0 +1,298 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package dsn contains the utilities used for dsn message (DSN) generation. +// +// It implements RFC 3464 and RFC 3462. +package dsn + +import ( + "errors" + "fmt" + "io" + "strings" + "text/template" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/dns" +) + +type ReportingMTAInfo struct { + ReportingMTA string + ReceivedFromMTA string + + // Message sender address, included as 'X-Maddy-Sender: rfc822; ADDR' field. + XSender string + + // Message identifier, included as 'X-Maddy-MsgId: MSGID' field. + XMessageID string + + // Time when message was enqueued for delivery by Reporting MTA. + ArrivalDate time.Time + + // Time when message delivery was attempted last time. + LastAttemptDate time.Time +} + +func (info ReportingMTAInfo) WriteTo(utf8 bool, w io.Writer) error { + // DSN format uses structure similar to MIME header, so we reuse + // MIME generator here. + h := textproto.Header{} + + if info.ReportingMTA == "" { + return errors.New("dsn: Reporting-MTA field is mandatory") + } + + reportingMTA, err := dns.SelectIDNA(utf8, info.ReportingMTA) + if err != nil { + return fmt.Errorf("dsn: cannot convert Reporting-MTA to a suitable representation: %w", err) + } + + h.Add("Reporting-MTA", "dns; "+reportingMTA) + + if info.ReceivedFromMTA != "" { + receivedFromMTA, err := dns.SelectIDNA(utf8, info.ReceivedFromMTA) + if err != nil { + return fmt.Errorf("dsn: cannot convert Received-From-MTA to a suitable representation: %w", err) + } + + h.Add("Received-From-MTA", "dns; "+receivedFromMTA) + } + + if info.XSender != "" { + sender, err := address.SelectIDNA(utf8, info.XSender) + if err != nil { + return fmt.Errorf("dsn: cannot convert X-Maddy-Sender to a suitable representation: %w", err) + } + + if utf8 { + h.Add("X-Maddy-Sender", "utf8; "+sender) + } else { + h.Add("X-Maddy-Sender", "rfc822; "+sender) + } + } + if info.XMessageID != "" { + h.Add("X-Maddy-MsgID", info.XMessageID) + } + + if !info.ArrivalDate.IsZero() { + h.Add("Arrival-Date", info.ArrivalDate.Format("Mon, 2 Jan 2006 15:04:05 -0700")) + } + if !info.ArrivalDate.IsZero() { + h.Add("Last-Attempt-Date", info.LastAttemptDate.Format("Mon, 2 Jan 2006 15:04:05 -0700")) + } + + return textproto.WriteHeader(w, h) +} + +type Action string + +const ( + ActionFailed Action = "failed" + ActionDelayed Action = "delayed" + ActionDelivered Action = "delivered" + ActionRelayed Action = "relayed" + ActionExpanded Action = "expanded" +) + +type RecipientInfo struct { + FinalRecipient string + RemoteMTA string + + Action Action + Status smtp.EnhancedCode + + // DiagnosticCode is the error that will be returned to the sender. + DiagnosticCode error +} + +func (info RecipientInfo) WriteTo(utf8 bool, w io.Writer) error { + // DSN format uses structure similar to MIME header, so we reuse + // MIME generator here. + h := textproto.Header{} + + if info.FinalRecipient == "" { + return errors.New("dsn: Final-Recipient is required") + } + finalRcpt, err := address.SelectIDNA(utf8, info.FinalRecipient) + if err != nil { + return fmt.Errorf("dsn: cannot convert Final-Recipient to a suitable representation: %w", err) + } + if utf8 { + h.Add("Final-Recipient", "utf8; "+finalRcpt) + } else { + h.Add("Final-Recipient", "rfc822; "+finalRcpt) + } + + if info.Action == "" { + return errors.New("dsn: Action is required") + } + h.Add("Action", string(info.Action)) + if info.Status[0] == 0 { + return errors.New("dsn: Status is required") + } + h.Add("Status", fmt.Sprintf("%d.%d.%d", info.Status[0], info.Status[1], info.Status[2])) + + if smtpErr, ok := info.DiagnosticCode.(*smtp.SMTPError); ok { + // Error message may contain newlines if it is received from another SMTP server. + // But we cannot directly insert CR/LF into Disagnostic-Code so rewrite it. + h.Add("Diagnostic-Code", fmt.Sprintf("smtp; %d %d.%d.%d %s", + smtpErr.Code, smtpErr.EnhancedCode[0], smtpErr.EnhancedCode[1], smtpErr.EnhancedCode[2], + strings.ReplaceAll(strings.ReplaceAll(smtpErr.Message, "\n", " "), "\r", " "))) + } else if utf8 { + // It might contain Unicode, so don't include it if we are not allowed to. + // ... I didn't bother implementing mangling logic to remove Unicode + // characters. + errorDesc := info.DiagnosticCode.Error() + errorDesc = strings.ReplaceAll(strings.ReplaceAll(errorDesc, "\n", " "), "\r", " ") + + h.Add("Diagnostic-Code", "X-Maddy; "+errorDesc) + } + + if info.RemoteMTA != "" { + remoteMTA, err := dns.SelectIDNA(utf8, info.RemoteMTA) + if err != nil { + return fmt.Errorf("dsn: cannot convert Remote-MTA to a suitable representation: %w", err) + } + + h.Add("Remote-MTA", "dns; "+remoteMTA) + } + + return textproto.WriteHeader(w, h) +} + +type Envelope struct { + MsgID string + From string + To string +} + +// GenerateDSN is a top-level function that should be used for generation of the DSNs. +// +// DSN header will be returned, body itself will be written to outWriter. +func GenerateDSN(utf8 bool, envelope Envelope, mtaInfo ReportingMTAInfo, rcptsInfo []RecipientInfo, failedHeader textproto.Header, outWriter io.Writer) (textproto.Header, error) { + partWriter := textproto.NewMultipartWriter(outWriter) + + reportHeader := textproto.Header{} + reportHeader.Add("Date", time.Now().Format("Mon, 2 Jan 2006 15:04:05 -0700")) + reportHeader.Add("Message-Id", envelope.MsgID) + reportHeader.Add("Content-Transfer-Encoding", "8bit") + reportHeader.Add("Content-Type", "multipart/report; report-type=delivery-status; boundary="+partWriter.Boundary()) + reportHeader.Add("MIME-Version", "1.0") + reportHeader.Add("Auto-Submitted", "auto-replied") + reportHeader.Add("To", envelope.To) + reportHeader.Add("From", envelope.From) + reportHeader.Add("Subject", "Undelivered Mail Returned to Sender") + + defer partWriter.Close() + + if err := writeHumanReadablePart(partWriter, mtaInfo, rcptsInfo); err != nil { + return textproto.Header{}, err + } + if err := writeMachineReadablePart(utf8, partWriter, mtaInfo, rcptsInfo); err != nil { + return textproto.Header{}, err + } + return reportHeader, writeHeader(utf8, partWriter, failedHeader) +} + +func writeHeader(utf8 bool, w *textproto.MultipartWriter, header textproto.Header) error { + partHeader := textproto.Header{} + partHeader.Add("Content-Description", "Undelivered message header") + if utf8 { + partHeader.Add("Content-Type", "message/global-headers") + } else { + partHeader.Add("Content-Type", "message/rfc822-headers") + } + partHeader.Add("Content-Transfer-Encoding", "8bit") + headerWriter, err := w.CreatePart(partHeader) + if err != nil { + return err + } + return textproto.WriteHeader(headerWriter, header) +} + +func writeMachineReadablePart(utf8 bool, w *textproto.MultipartWriter, mtaInfo ReportingMTAInfo, rcptsInfo []RecipientInfo) error { + machineHeader := textproto.Header{} + if utf8 { + machineHeader.Add("Content-Type", "message/global-delivery-status") + } else { + machineHeader.Add("Content-Type", "message/delivery-status") + } + machineHeader.Add("Content-Description", "Delivery report") + machineWriter, err := w.CreatePart(machineHeader) + if err != nil { + return err + } + + // WriteTo will add an empty line after output. + if err := mtaInfo.WriteTo(utf8, machineWriter); err != nil { + return err + } + + for _, rcpt := range rcptsInfo { + if err := rcpt.WriteTo(utf8, machineWriter); err != nil { + return err + } + } + return nil +} + +// failedText is the text of the human-readable part of DSN. +var failedText = template.Must(template.New("dsn-text").Parse(` +This is the mail delivery system at {{.ReportingMTA}}. + +Unfortunately, your message could not be delivered to one or more +recipients. The usual cause of this problem is invalid +recipient address or maintenance at the recipient side. + +Contact the postmaster for further assistance, provide the Message ID (below): + +Message ID: {{.XMessageID}} +Arrival: {{.ArrivalDate}} +Last delivery attempt: {{.LastAttemptDate}} + +`)) + +func writeHumanReadablePart(w *textproto.MultipartWriter, mtaInfo ReportingMTAInfo, rcptsInfo []RecipientInfo) error { + humanHeader := textproto.Header{} + humanHeader.Add("Content-Transfer-Encoding", "8bit") + humanHeader.Add("Content-Type", `text/plain; charset="utf-8"`) + humanHeader.Add("Content-Description", "Notification") + humanWriter, err := w.CreatePart(humanHeader) + if err != nil { + return err + } + + mtaInfo.ArrivalDate = mtaInfo.ArrivalDate.Truncate(time.Second) + mtaInfo.LastAttemptDate = mtaInfo.LastAttemptDate.Truncate(time.Second) + + if err := failedText.Execute(humanWriter, mtaInfo); err != nil { + return err + } + + for _, rcpt := range rcptsInfo { + if _, err := fmt.Fprintf(humanWriter, "Delivery to %s failed with error: %v\n", rcpt.FinalRecipient, rcpt.DiagnosticCode); err != nil { + return err + } + } + + return nil +} diff --git a/internal/endpoint/dovecot_sasld/dovecot_sasl.go b/internal/endpoint/dovecot_sasld/dovecot_sasl.go new file mode 100644 index 0000000..2679696 --- /dev/null +++ b/internal/endpoint/dovecot_sasld/dovecot_sasl.go @@ -0,0 +1,127 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dovecotsasld + +import ( + "fmt" + stdlog "log" + "net" + "strings" + "sync" + + "github.com/emersion/go-sasl" + dovecotsasl "github.com/foxcpp/go-dovecot-sasl" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth" + "github.com/foxcpp/maddy/internal/authz" +) + +const modName = "dovecot_sasld" + +type Endpoint struct { + addrs []string + log log.Logger + saslAuth auth.SASLAuth + + listenersWg sync.WaitGroup + + srv *dovecotsasl.Server +} + +func New(_ string, addrs []string) (module.Module, error) { + return &Endpoint{ + addrs: addrs, + saslAuth: auth.SASLAuth{ + Log: log.Logger{Name: modName + "/saslauth"}, + }, + log: log.Logger{Name: modName, Debug: log.DefaultLogger.Debug}, + }, nil +} + +func (endp *Endpoint) Name() string { + return modName +} + +func (endp *Endpoint) InstanceName() string { + return modName +} + +func (endp *Endpoint) Init(cfg *config.Map) error { + cfg.Callback("auth", func(m *config.Map, node config.Node) error { + return endp.saslAuth.AddProvider(m, node) + }) + cfg.Bool("sasl_login", false, false, &endp.saslAuth.EnableLogin) + config.EnumMapped(cfg, "auth_map_normalize", true, false, authz.NormalizeFuncs, authz.NormalizeAuto, + &endp.saslAuth.AuthNormalize) + modconfig.Table(cfg, "auth_map", true, false, nil, &endp.saslAuth.AuthMap) + if _, err := cfg.Process(); err != nil { + return err + } + + endp.srv = dovecotsasl.NewServer() + endp.srv.Log = stdlog.New(endp.log, "", 0) + endp.saslAuth.Log.Debug = endp.log.Debug + + for _, mech := range endp.saslAuth.SASLMechanisms() { + endp.srv.AddMechanism(mech, mechInfo[mech], func(req *dovecotsasl.AuthReq) sasl.Server { + var remoteAddr net.Addr + if req.RemoteIP != nil && req.RemotePort != 0 { + remoteAddr = &net.TCPAddr{IP: req.RemoteIP, Port: int(req.RemotePort)} + } + + return endp.saslAuth.CreateSASL(mech, remoteAddr, func(_ string, _ auth.ContextData) error { return nil }) + }) + } + + for _, addr := range endp.addrs { + parsed, err := config.ParseEndpoint(addr) + if err != nil { + return fmt.Errorf("%s: %v", modName, err) + } + + l, err := net.Listen(parsed.Network(), parsed.Address()) + if err != nil { + return fmt.Errorf("%s: %v", modName, err) + } + endp.log.Printf("listening on %v", l.Addr()) + + endp.listenersWg.Add(1) + go func() { + defer endp.listenersWg.Done() + if err := endp.srv.Serve(l); err != nil { + if !strings.HasSuffix(err.Error(), "use of closed network connection") { + endp.log.Printf("failed to serve %v: %v", l.Addr(), err) + } + } + }() + } + + return nil +} + +func (endp *Endpoint) Close() error { + return endp.srv.Close() +} + +func init() { + module.RegisterEndpoint(modName, New) +} diff --git a/internal/endpoint/dovecot_sasld/mech_info.go b/internal/endpoint/dovecot_sasld/mech_info.go new file mode 100644 index 0000000..ab7b5f8 --- /dev/null +++ b/internal/endpoint/dovecot_sasld/mech_info.go @@ -0,0 +1,33 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dovecotsasld + +import ( + "github.com/emersion/go-sasl" + dovecotsasl "github.com/foxcpp/go-dovecot-sasl" +) + +var mechInfo = map[string]dovecotsasl.Mechanism{ + sasl.Plain: { + Plaintext: true, + }, + sasl.Login: { + Plaintext: true, + }, +} diff --git a/internal/endpoint/imap/imap.go b/internal/endpoint/imap/imap.go new file mode 100644 index 0000000..cece797 --- /dev/null +++ b/internal/endpoint/imap/imap.go @@ -0,0 +1,322 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imap + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "strings" + "sync" + + "github.com/emersion/go-imap" + compress "github.com/emersion/go-imap-compress" + sortthread "github.com/emersion/go-imap-sortthread" + imapbackend "github.com/emersion/go-imap/backend" + imapserver "github.com/emersion/go-imap/server" + "github.com/emersion/go-message" + _ "github.com/emersion/go-message/charset" + "github.com/emersion/go-sasl" + i18nlevel "github.com/foxcpp/go-imap-i18nlevel" + namespace "github.com/foxcpp/go-imap-namespace" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + tls2 "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth" + "github.com/foxcpp/maddy/internal/authz" + "github.com/foxcpp/maddy/internal/proxy_protocol" + "github.com/foxcpp/maddy/internal/updatepipe" +) + +type Endpoint struct { + addrs []string + serv *imapserver.Server + listeners []net.Listener + proxyProtocol *proxy_protocol.ProxyProtocol + Store module.Storage + + tlsConfig *tls.Config + listenersWg sync.WaitGroup + + saslAuth auth.SASLAuth + + storageNormalize authz.NormalizeFunc + storageMap module.Table + + Log log.Logger +} + +func New(modName string, addrs []string) (module.Module, error) { + endp := &Endpoint{ + addrs: addrs, + Log: log.Logger{Name: modName}, + saslAuth: auth.SASLAuth{ + Log: log.Logger{Name: modName + "/sasl"}, + }, + } + + return endp, nil +} + +func (endp *Endpoint) Init(cfg *config.Map) error { + var ( + insecureAuth bool + ioDebug bool + ioErrors bool + ) + + cfg.Callback("auth", func(m *config.Map, node config.Node) error { + return endp.saslAuth.AddProvider(m, node) + }) + cfg.Bool("sasl_login", false, false, &endp.saslAuth.EnableLogin) + cfg.Custom("storage", false, true, nil, modconfig.StorageDirective, &endp.Store) + cfg.Custom("tls", true, true, nil, tls2.TLSDirective, &endp.tlsConfig) + cfg.Custom("proxy_protocol", false, false, nil, proxy_protocol.ProxyProtocolDirective, &endp.proxyProtocol) + cfg.Bool("insecure_auth", false, false, &insecureAuth) + cfg.Bool("io_debug", false, false, &ioDebug) + cfg.Bool("io_errors", false, false, &ioErrors) + cfg.Bool("debug", true, false, &endp.Log.Debug) + config.EnumMapped(cfg, "storage_map_normalize", false, false, authz.NormalizeFuncs, authz.NormalizeAuto, + &endp.storageNormalize) + modconfig.Table(cfg, "storage_map", false, false, nil, &endp.storageMap) + config.EnumMapped(cfg, "auth_map_normalize", true, false, authz.NormalizeFuncs, authz.NormalizeAuto, + &endp.saslAuth.AuthNormalize) + modconfig.Table(cfg, "auth_map", true, false, nil, &endp.saslAuth.AuthMap) + if _, err := cfg.Process(); err != nil { + return err + } + + if updBe, ok := endp.Store.(updatepipe.Backend); ok { + if err := updBe.EnableUpdatePipe(updatepipe.ModeReplicate); err != nil { + endp.Log.Error("failed to initialize updates pipe", err) + } + } + + endp.saslAuth.Log.Debug = endp.Log.Debug + + addresses := make([]config.Endpoint, 0, len(endp.addrs)) + for _, addr := range endp.addrs { + saddr, err := config.ParseEndpoint(addr) + if err != nil { + return fmt.Errorf("imap: invalid address: %s", addr) + } + addresses = append(addresses, saddr) + } + + endp.serv = imapserver.New(endp) + endp.serv.AllowInsecureAuth = insecureAuth + endp.serv.TLSConfig = endp.tlsConfig + if ioErrors { + endp.serv.ErrorLog = &endp.Log + } else { + endp.serv.ErrorLog = log.Logger{Out: log.NopOutput{}} + } + if ioDebug { + endp.serv.Debug = endp.Log.DebugWriter() + endp.Log.Println("I/O debugging is on! It may leak passwords in logs, be careful!") + } + + if err := endp.enableExtensions(); err != nil { + return err + } + + for _, mech := range endp.saslAuth.SASLMechanisms() { + endp.serv.EnableAuth(mech, func(c imapserver.Conn) sasl.Server { + return endp.saslAuth.CreateSASL(mech, c.Info().RemoteAddr, func(identity string, data auth.ContextData) error { + return endp.openAccount(c, identity) + }) + }) + } + + return endp.setupListeners(addresses) +} + +func (endp *Endpoint) setupListeners(addresses []config.Endpoint) error { + for _, addr := range addresses { + var l net.Listener + var err error + l, err = net.Listen(addr.Network(), addr.Address()) + if err != nil { + return fmt.Errorf("imap: %v", err) + } + endp.Log.Printf("listening on %v", addr) + + if addr.IsTLS() { + if endp.tlsConfig == nil { + return errors.New("imap: can't bind on IMAPS endpoint without TLS configuration") + } + l = tls.NewListener(l, endp.tlsConfig) + } + + if endp.proxyProtocol != nil { + l = proxy_protocol.NewListener(l, endp.proxyProtocol, endp.Log) + } + + endp.listeners = append(endp.listeners, l) + + endp.listenersWg.Add(1) + go func() { + if err := endp.serv.Serve(l); err != nil && !strings.HasSuffix(err.Error(), "use of closed network connection") { + endp.Log.Printf("imap: failed to serve %s: %s", addr, err) + } + endp.listenersWg.Done() + }() + } + + if endp.serv.AllowInsecureAuth { + endp.Log.Println("authentication over unencrypted connections is allowed, this is insecure configuration and should be used only for testing!") + } + if endp.serv.TLSConfig == nil { + endp.Log.Println("TLS is disabled, this is insecure configuration and should be used only for testing!") + endp.serv.AllowInsecureAuth = true + } + + return nil +} + +func (endp *Endpoint) Name() string { + return "imap" +} + +func (endp *Endpoint) InstanceName() string { + return "imap" +} + +func (endp *Endpoint) Close() error { + for _, l := range endp.listeners { + l.Close() + } + if err := endp.serv.Close(); err != nil { + return err + } + endp.listenersWg.Wait() + return nil +} + +func (endp *Endpoint) usernameForStorage(ctx context.Context, saslUsername string) (string, error) { + saslUsername, err := endp.storageNormalize(saslUsername) + if err != nil { + return "", err + } + + if endp.storageMap == nil { + return saslUsername, nil + } + + mapped, ok, err := endp.storageMap.Lookup(ctx, saslUsername) + if err != nil { + return "", err + } + if !ok { + return "", imapbackend.ErrInvalidCredentials + } + + if saslUsername != mapped { + endp.Log.DebugMsg("using mapped username for storage", "username", saslUsername, "mapped_username", mapped) + } + + return mapped, nil +} + +func (endp *Endpoint) openAccount(c imapserver.Conn, identity string) error { + username, err := endp.usernameForStorage(context.TODO(), identity) + if err != nil { + if errors.Is(err, imapbackend.ErrInvalidCredentials) { + return err + } + endp.Log.Error("failed to determine storage account name", err, "username", username) + return fmt.Errorf("internal server error") + } + + u, err := endp.Store.GetOrCreateIMAPAcct(username) + if err != nil { + return err + } + ctx := c.Context() + ctx.State = imap.AuthenticatedState + ctx.User = u + return nil +} + +func (endp *Endpoint) Login(connInfo *imap.ConnInfo, username, password string) (imapbackend.User, error) { + // saslAuth handles AuthMap calling. + err := endp.saslAuth.AuthPlain(username, password) + if err != nil { + endp.Log.Error("authentication failed", err, "username", username, "src_ip", connInfo.RemoteAddr) + return nil, imapbackend.ErrInvalidCredentials + } + + storageUsername, err := endp.usernameForStorage(context.TODO(), username) + if err != nil { + if errors.Is(err, imapbackend.ErrInvalidCredentials) { + return nil, err + } + endp.Log.Error("authentication failed due to an internal error", err, "username", username, "src_ip", connInfo.RemoteAddr) + return nil, fmt.Errorf("internal server error") + } + + return endp.Store.GetOrCreateIMAPAcct(storageUsername) +} + +func (endp *Endpoint) I18NLevel() int { + be, ok := endp.Store.(i18nlevel.Backend) + if !ok { + return 0 + } + return be.I18NLevel() +} + +func (endp *Endpoint) enableExtensions() error { + exts := endp.Store.IMAPExtensions() + for _, ext := range exts { + switch ext { + case "I18NLEVEL=1", "I18NLEVEL=2": + endp.serv.Enable(i18nlevel.NewExtension()) + case "SORT": + endp.serv.Enable(sortthread.NewSortExtension()) + } + if strings.HasPrefix(ext, "THREAD") { + endp.serv.Enable(sortthread.NewThreadExtension()) + } + } + + endp.serv.Enable(compress.NewExtension()) + endp.serv.Enable(namespace.NewExtension()) + + return nil +} + +func (endp *Endpoint) SupportedThreadAlgorithms() []sortthread.ThreadAlgorithm { + be, ok := endp.Store.(sortthread.ThreadBackend) + if !ok { + return nil + } + + return be.SupportedThreadAlgorithms() +} + +func init() { + module.RegisterEndpoint("imap", New) + + imap.CharsetReader = message.CharsetReader +} diff --git a/internal/endpoint/openmetrics/om.go b/internal/endpoint/openmetrics/om.go new file mode 100644 index 0000000..874a333 --- /dev/null +++ b/internal/endpoint/openmetrics/om.go @@ -0,0 +1,107 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package openmetrics + +import ( + "errors" + "fmt" + "net" + "net/http" + "sync" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +const modName = "openmetrics" + +type Endpoint struct { + addrs []string + logger log.Logger + + listenersWg sync.WaitGroup + serv http.Server + mux *http.ServeMux +} + +func New(_ string, args []string) (module.Module, error) { + return &Endpoint{ + addrs: args, + logger: log.Logger{Name: modName, Debug: log.DefaultLogger.Debug}, + }, nil +} + +func (e *Endpoint) Init(cfg *config.Map) error { + cfg.Bool("debug", false, false, &e.logger.Debug) + if _, err := cfg.Process(); err != nil { + return err + } + + e.mux = http.NewServeMux() + e.mux.Handle("/metrics", promhttp.Handler()) + e.serv.Handler = e.mux + + for _, a := range e.addrs { + endp, err := config.ParseEndpoint(a) + if err != nil { + return fmt.Errorf("%s: malformed endpoint: %v", modName, err) + } + if endp.IsTLS() { + return fmt.Errorf("%s: TLS is not supported yet", modName) + } + l, err := net.Listen(endp.Network(), endp.Address()) + if err != nil { + return fmt.Errorf("%s: %v", modName, err) + } + + e.listenersWg.Add(1) + go func() { + e.logger.Println("listening on", endp.String()) + err := e.serv.Serve(l) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + e.logger.Error("serve failed", err, "endpoint", a) + } + e.listenersWg.Done() + }() + } + + return nil +} + +func (e *Endpoint) Name() string { + return modName +} + +func (e *Endpoint) InstanceName() string { + return "" +} + +func (e *Endpoint) Close() error { + if err := e.serv.Close(); err != nil { + return err + } + e.listenersWg.Wait() + return nil +} + +func init() { + module.RegisterEndpoint(modName, New) +} diff --git a/internal/endpoint/smtp/date.go b/internal/endpoint/smtp/date.go new file mode 100644 index 0000000..e41daa1 --- /dev/null +++ b/internal/endpoint/smtp/date.go @@ -0,0 +1,66 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import ( + "fmt" + "regexp" + "time" +) + +// Taken from https://github.com/emersion/go-imap/blob/09c1d69/date.go. + +var dateTimeLayouts = [...]string{ + // Defined in RFC 5322 section 3.3, mentioned as env-date in RFC 3501 page 84. + "Mon, 02 Jan 2006 15:04:05 -0700", + "_2 Jan 2006 15:04:05 -0700", + "_2 Jan 2006 15:04:05 MST", + "_2 Jan 2006 15:04 -0700", + "_2 Jan 2006 15:04 MST", + "_2 Jan 06 15:04:05 -0700", + "_2 Jan 06 15:04:05 MST", + "_2 Jan 06 15:04 -0700", + "_2 Jan 06 15:04 MST", + "Mon, _2 Jan 2006 15:04:05 -0700", + "Mon, _2 Jan 2006 15:04:05 MST", + "Mon, _2 Jan 2006 15:04 -0700", + "Mon, _2 Jan 2006 15:04 MST", + "Mon, _2 Jan 06 15:04:05 -0700", + "Mon, _2 Jan 06 15:04:05 MST", + "Mon, _2 Jan 06 15:04 -0700", + "Mon, _2 Jan 06 15:04 MST", +} + +// TODO: this is a blunt way to strip any trailing CFWS (comment). A sharper +// one would strip multiple CFWS, and only if really valid according to +// RFC5322. +var commentRE = regexp.MustCompile(`[ \t]+\(.*\)$`) + +// Try parsing the date based on the layouts defined in RFC 5322, section 3.3. +// Inspired by https://github.com/golang/go/blob/master/src/net/mail/message.go +func parseMessageDateTime(maybeDate string) (time.Time, error) { + maybeDate = commentRE.ReplaceAllString(maybeDate, "") + for _, layout := range dateTimeLayouts { + parsed, err := time.Parse(layout, maybeDate) + if err == nil { + return parsed, nil + } + } + return time.Time{}, fmt.Errorf("date %s could not be parsed", maybeDate) +} diff --git a/internal/endpoint/smtp/metrics.go b/internal/endpoint/smtp/metrics.go new file mode 100644 index 0000000..509241b --- /dev/null +++ b/internal/endpoint/smtp/metrics.go @@ -0,0 +1,87 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import "github.com/prometheus/client_golang/prometheus" + +var ( + startedSMTPTransactions = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "smtp", + Name: "started_transactions", + Help: "Amount of SMTP trasanactions started", + }, + []string{"module"}, + ) + completedSMTPTransactions = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "smtp", + Name: "smtp_completed_transactions", + Help: "Amount of SMTP trasanactions successfully completed", + }, + []string{"module"}, + ) + abortedSMTPTransactions = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "smtp", + Name: "aborted_transactions", + Help: "Amount of SMTP trasanactions aborted", + }, + []string{"module"}, + ) + + ratelimitDefers = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "smtp", + Name: "ratelimit_deferred", + Help: "Messages rejected with 4xx code due to ratelimiting", + }, + []string{"module"}, + ) + failedLogins = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "smtp", + Name: "failed_logins", + Help: "AUTH command failures", + }, + []string{"module"}, + ) + failedCmds = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "smtp", + Name: "failed_commands", + Help: "Failed transaction commands (MAIL, RCPT, DATA)", + }, + []string{"module", "command", "smtp_code", "smtp_enchcode"}, + ) +) + +func init() { + prometheus.MustRegister(startedSMTPTransactions) + prometheus.MustRegister(completedSMTPTransactions) + prometheus.MustRegister(abortedSMTPTransactions) + prometheus.MustRegister(ratelimitDefers) + prometheus.MustRegister(failedCmds) +} diff --git a/internal/endpoint/smtp/session.go b/internal/endpoint/smtp/session.go new file mode 100644 index 0000000..cff1c01 --- /dev/null +++ b/internal/endpoint/smtp/session.go @@ -0,0 +1,656 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "net" + "runtime/trace" + "strconv" + "strings" + "sync" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-sasl" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth" +) + +func limitReader(r io.Reader, n int64, err error) *limitedReader { + return &limitedReader{R: r, N: n, E: err, Enabled: true} +} + +type limitedReader struct { + R io.Reader + N int64 + E error + Enabled bool +} + +// same as io.LimitedReader.Read except returning the custom error and the option +// to be disabled +func (l *limitedReader) Read(p []byte) (n int, err error) { + if !l.Enabled { + return l.R.Read(p) + } + if l.N <= 0 { + return 0, l.E + } + if int64(len(p)) > l.N { + p = p[0:l.N] + } + n, err = l.R.Read(p) + l.N -= int64(n) + return +} + +type Session struct { + endp *Endpoint + + // Specific for this session. + // sessionCtx is not used for cancellation or timeouts, only for tracing. + sessionCtx context.Context + cancelRDNS func() + connState module.ConnState + repeatedMailErrs int + loggedRcptErrors int + + // Specific for the currently handled message. + // msgCtx is not used for cancellation or timeouts, only for tracing. + // It is the subcontext of sessionCtx. + // Mutex is used to prevent Close from accessing inconsistent state when it + // is called asynchronously to any SMTP command. + msgLock sync.Mutex + msgCtx context.Context + msgTask *trace.Task + mailFrom string + opts smtp.MailOptions + msgMeta *module.MsgMetadata + delivery module.Delivery + deliveryErr error + + log log.Logger +} + +func (s *Session) AuthMechanisms() []string { + return s.endp.saslAuth.SASLMechanisms() +} + +func (s *Session) Auth(mech string) (sasl.Server, error) { + return s.endp.saslAuth.CreateSASL(mech, s.connState.RemoteAddr, func(identity string, data auth.ContextData) error { + s.connState.AuthUser = identity + s.connState.AuthPassword = data.Password + return nil + }), nil +} + +func (s *Session) Reset() { + s.msgLock.Lock() + defer s.msgLock.Unlock() + + if s.delivery != nil { + s.abort(s.msgCtx) + } + s.endp.Log.DebugMsg("reset") +} + +func (s *Session) releaseLimits() { + domain := "" + if s.mailFrom != "" { + var err error + _, domain, err = address.Split(s.mailFrom) + if err != nil { + return + } + } + + addr, ok := s.msgMeta.Conn.RemoteAddr.(*net.TCPAddr) + if !ok { + addr = &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)} + } + s.endp.limits.ReleaseMsg(addr.IP, domain) +} + +func (s *Session) abort(ctx context.Context) { + if err := s.delivery.Abort(ctx); err != nil { + s.endp.Log.Error("delivery abort failed", err) + } + s.log.Msg("aborted", "msg_id", s.msgMeta.ID) + abortedSMTPTransactions.WithLabelValues(s.endp.name).Inc() + s.cleanSession() +} + +func (s *Session) cleanSession() { + s.releaseLimits() + + s.mailFrom = "" + s.opts = smtp.MailOptions{} + s.msgMeta = nil + s.delivery = nil + s.deliveryErr = nil + s.msgCtx = nil + s.msgTask.End() +} + +func (s *Session) AuthPlain(username, password string) error { + // Executed before authentication and session initialization. + if err := s.endp.pipeline.RunEarlyChecks(context.TODO(), &s.connState); err != nil { + return s.endp.wrapErr("", true, "AUTH", err) + } + + // saslAuth will handle AuthMap and AuthNormalize. + err := s.endp.saslAuth.AuthPlain(username, password) + if err != nil { + s.endp.Log.Error("authentication failed", err, "username", username, "src_ip", s.connState.RemoteAddr) + + failedLogins.WithLabelValues(s.endp.name).Inc() + + if exterrors.IsTemporary(err) { + return &smtp.SMTPError{ + Code: 454, + EnhancedCode: smtp.EnhancedCode{4, 7, 0}, + Message: "Temporary authentication failure", + } + } + + return &smtp.SMTPError{ + Code: 535, + EnhancedCode: smtp.EnhancedCode{5, 7, 8}, + Message: "Invalid credentials", + } + } + + s.connState.AuthUser = username + s.connState.AuthPassword = password + + return nil +} + +func (s *Session) startDelivery(ctx context.Context, from string, opts smtp.MailOptions) (string, error) { + var err error + msgMeta := &module.MsgMetadata{ + Conn: &s.connState, + SMTPOpts: opts, + } + msgMeta.ID, err = module.GenerateMsgID() + if err != nil { + return "", err + } + + if s.connState.AuthUser != "" { + s.log.Msg("incoming message", + "src_host", msgMeta.Conn.Hostname, + "src_ip", msgMeta.Conn.RemoteAddr.String(), + "sender", from, + "msg_id", msgMeta.ID, + "username", s.connState.AuthUser, + ) + } else { + s.log.Msg("incoming message", + "src_host", msgMeta.Conn.Hostname, + "src_ip", msgMeta.Conn.RemoteAddr.String(), + "sender", from, + "msg_id", msgMeta.ID, + ) + } + + // INTERNATIONALIZATION: Do not permit non-ASCII addresses unless SMTPUTF8 is + // used. + if !opts.UTF8 { + for _, ch := range from { + if ch > 128 { + return "", &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 6, 7}, + Message: "SMTPUTF8 is required for non-ASCII senders", + } + } + } + } + + // Decode punycode, normalize to NFC and case-fold address. + cleanFrom := from + if from != "" { + cleanFrom, err = address.CleanDomain(from) + if err != nil { + return "", &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 1, 7}, + Message: "Unable to normalize the sender address", + } + } + } + + msgMeta.OriginalFrom = from + + domain := "" + if cleanFrom != "" { + _, domain, err = address.Split(cleanFrom) + if err != nil { + return "", err + } + } + remoteIP, ok := msgMeta.Conn.RemoteAddr.(*net.TCPAddr) + if !ok { + remoteIP = &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)} + } + if err := s.endp.limits.TakeMsg(context.Background(), remoteIP.IP, domain); err != nil { + return "", err + } + + s.msgCtx, s.msgTask = trace.NewTask(ctx, "Incoming Message") + + mailCtx, mailTask := trace.NewTask(s.msgCtx, "MAIL FROM") + defer mailTask.End() + + delivery, err := s.endp.pipeline.Start(mailCtx, msgMeta, cleanFrom) + if err != nil { + s.msgCtx = nil + s.msgTask.End() + s.endp.limits.ReleaseMsg(remoteIP.IP, domain) + return msgMeta.ID, err + } + + startedSMTPTransactions.WithLabelValues(s.endp.name).Inc() + + s.msgMeta = msgMeta + s.mailFrom = cleanFrom + s.delivery = delivery + + return msgMeta.ID, nil +} + +func (s *Session) Mail(from string, opts *smtp.MailOptions) error { + if s.endp.authAlwaysRequired && s.connState.AuthUser == "" { + return smtp.ErrAuthRequired + } + + s.msgLock.Lock() + defer s.msgLock.Unlock() + + if !s.endp.deferServerReject { + // Will initialize s.msgCtx. + msgID, err := s.startDelivery(s.sessionCtx, from, *opts) + if err != nil { + if !errors.Is(err, context.DeadlineExceeded) { + s.log.Error("MAIL FROM error", err, "msg_id", msgID) + } + return s.endp.wrapErr(msgID, !opts.UTF8, "MAIL", err) + } + } + + // Keep the MAIL FROM argument for deferred startDelivery. + s.mailFrom = from + s.opts = *opts + + return nil +} + +func (s *Session) fetchRDNSName(ctx context.Context) { + defer trace.StartRegion(ctx, "rDNS fetch").End() + + tcpAddr, ok := s.connState.RemoteAddr.(*net.TCPAddr) + if !ok { + s.connState.RDNSName.Set(nil, nil) + return + } + + name, err := dns.LookupAddr(ctx, s.endp.resolver, tcpAddr.IP) + if err != nil { + dnsErr, ok := err.(*net.DNSError) + if ok && dnsErr.IsNotFound { + s.connState.RDNSName.Set(nil, nil) + return + } + + if !errors.Is(err, context.Canceled) { + // Often occurs when transaction completes before rDNS lookup and + // rDNS name was not actually needed. So do not log cancelation + // error if that's the case. + + reason, misc := exterrors.UnwrapDNSErr(err) + misc["reason"] = reason + s.log.Error("rDNS error", exterrors.WithFields(err, misc), "src_ip", s.connState.RemoteAddr) + } + s.connState.RDNSName.Set(nil, err) + return + } + + s.connState.RDNSName.Set(name, nil) +} + +func (s *Session) Rcpt(to string, opts *smtp.RcptOptions) error { + s.msgLock.Lock() + defer s.msgLock.Unlock() + + // deferServerReject = true and this is the first RCPT TO command. + if s.delivery == nil { + // If we already attempted to initialize the delivery - + // fail again. + if s.deliveryErr != nil { + s.repeatedMailErrs++ + // The deliveryErr is already wrapped. + return s.deliveryErr + } + + // It will initialize s.msgCtx. + msgID, err := s.startDelivery(s.sessionCtx, s.mailFrom, s.opts) + if err != nil { + if !errors.Is(err, context.DeadlineExceeded) { + s.log.Error("MAIL FROM error (deferred)", err, "rcpt", to, "msg_id", msgID) + } + s.deliveryErr = s.endp.wrapErr(msgID, !s.opts.UTF8, "RCPT", err) + return s.deliveryErr + } + } + + rcptCtx, rcptTask := trace.NewTask(s.msgCtx, "RCPT TO") + defer rcptTask.End() + + if err := s.rcpt(rcptCtx, to, opts); err != nil { + if s.loggedRcptErrors < s.endp.maxLoggedRcptErrors { + s.log.Error("RCPT error", err, "rcpt", to, "msg_id", s.msgMeta.ID) + s.loggedRcptErrors++ + if s.loggedRcptErrors == s.endp.maxLoggedRcptErrors { + s.log.Msg("too many RCPT errors, possible dictonary attack", "src_ip", s.connState.RemoteAddr, "msg_id", s.msgMeta.ID) + } + } + return s.endp.wrapErr(s.msgMeta.ID, !s.opts.UTF8, "RCPT", err) + } + s.endp.Log.Msg("RCPT ok", "rcpt", to, "msg_id", s.msgMeta.ID) + return nil +} + +func (s *Session) rcpt(ctx context.Context, to string, opts *smtp.RcptOptions) error { + // INTERNATIONALIZATION: Do not permit non-ASCII addresses unless SMTPUTF8 is + // used. + if !address.IsASCII(to) && !s.opts.UTF8 { + return &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 6, 7}, + Message: "SMTPUTF8 is required for non-ASCII recipients", + } + } + cleanTo, err := address.CleanDomain(to) + if err != nil { + return &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 2}, + Message: "Unable to normalize the recipient address", + } + } + + return s.delivery.AddRcpt(ctx, cleanTo, *opts) +} + +func (s *Session) Logout() error { + s.msgLock.Lock() + defer s.msgLock.Unlock() + + if s.delivery != nil { + s.abort(s.msgCtx) + + if s.repeatedMailErrs > s.endp.maxLoggedRcptErrors { + s.log.Msg("MAIL FROM repeated error a lot of times, possible dictonary attack", "count", s.repeatedMailErrs, "src_ip", s.connState.RemoteAddr) + } + } + if s.cancelRDNS != nil { + s.cancelRDNS() + } + + s.endp.sessionCnt.Add(-1) + + return nil +} + +func (s *Session) prepareBody(r io.Reader) (textproto.Header, buffer.Buffer, error) { + limitr := limitReader(r, s.endp.maxHeaderBytes, &exterrors.SMTPError{ + Code: 552, + EnhancedCode: exterrors.EnhancedCode{5, 3, 4}, + Message: "Message header size exceeds limit", + }) + + bufr := bufio.NewReader(limitr) + header, err := textproto.ReadHeader(bufr) + if err != nil { + return textproto.Header{}, nil, fmt.Errorf("I/O error while parsing header: %w", err) + } + + if s.endp.submission { + // The MsgMetadata is passed by pointer all the way down. + if err := s.submissionPrepare(s.msgMeta, &header); err != nil { + return textproto.Header{}, nil, err + } + } + + // the header size check is done. The message size will be checked by go-smtp + limitr.Enabled = false + + buf, err := s.endp.buffer(bufr) + if err != nil { + return textproto.Header{}, nil, fmt.Errorf("I/O error while writing buffer: %w", err) + } + + return header, buf, nil +} + +func (s *Session) Data(r io.Reader) error { + s.msgLock.Lock() + defer s.msgLock.Unlock() + + bodyCtx, bodyTask := trace.NewTask(s.msgCtx, "DATA") + defer bodyTask.End() + + wrapErr := func(err error) error { + s.log.Error("DATA error", err, "msg_id", s.msgMeta.ID) + return s.endp.wrapErr(s.msgMeta.ID, !s.opts.UTF8, "DATA", err) + } + + header, buf, err := s.prepareBody(r) + if err != nil { + return wrapErr(err) + } + defer func() { + if err := buf.Remove(); err != nil { + s.log.Error("failed to remove buffered body", err) + } + + // go-smtp will call Reset, but it will call Abort if delivery is non-nil. + s.cleanSession() + }() + + if err := s.checkRoutingLoops(header); err != nil { + return wrapErr(err) + } + + if strings.EqualFold(header.Get("TLS-Required"), "No") { + s.msgMeta.TLSRequireOverride = true + } + + if err := s.delivery.Body(bodyCtx, header, buf); err != nil { + return wrapErr(err) + } + + if err := s.delivery.Commit(bodyCtx); err != nil { + return wrapErr(err) + } + + s.log.Msg("accepted", "msg_id", s.msgMeta.ID) + + return nil +} + +type statusWrapper struct { + sc smtp.StatusCollector + s *Session +} + +func (sw statusWrapper) SetStatus(rcpt string, err error) { + sw.sc.SetStatus(rcpt, sw.s.endp.wrapErr(sw.s.msgMeta.ID, !sw.s.opts.UTF8, "DATA", err)) +} + +func (s *Session) LMTPData(r io.Reader, sc smtp.StatusCollector) error { + s.msgLock.Lock() + defer s.msgLock.Unlock() + + bodyCtx, bodyTask := trace.NewTask(s.msgCtx, "DATA") + defer bodyTask.End() + + wrapErr := func(err error) error { + s.log.Error("DATA error", err, "msg_id", s.msgMeta.ID) + return s.endp.wrapErr(s.msgMeta.ID, !s.opts.UTF8, "DATA", err) + } + + header, buf, err := s.prepareBody(r) + if err != nil { + return wrapErr(err) + } + defer func() { + if err := buf.Remove(); err != nil { + s.log.Error("failed to remove buffered body", err) + } + + // go-smtp will call Reset, but it will call Abort if delivery is non-nil. + s.cleanSession() + }() + + if strings.EqualFold(header.Get("TLS-Required"), "No") { + s.msgMeta.TLSRequireOverride = true + } + + if err := s.checkRoutingLoops(header); err != nil { + return wrapErr(err) + } + + s.delivery.(module.PartialDelivery).BodyNonAtomic(bodyCtx, statusWrapper{sc, s}, header, buf) + + // We can't really tell whether it is failed completely or succeeded + // so always commit. Should be harmless, anyway. + if err := s.delivery.Commit(bodyCtx); err != nil { + return wrapErr(err) + } + + s.log.Msg("accepted", "msg_id", s.msgMeta.ID) + + return nil +} + +func (s *Session) checkRoutingLoops(header textproto.Header) error { + // RFC 5321 Section 6.3: + // >Simple counting of the number of "Received:" header fields in a + // >message has proven to be an effective, although rarely optimal, + // >method of detecting loops in mail systems. + receivedCount := 0 + for f := header.FieldsByKey("Received"); f.Next(); { + receivedCount++ + } + if receivedCount > s.endp.maxReceived { + return &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 4, 6}, + Message: fmt.Sprintf("Too many Received header fields (%d), possible forwarding loop", receivedCount), + } + } + + return nil +} + +func (endp *Endpoint) wrapErr(msgId string, mangleUTF8 bool, command string, err error) error { + if err == nil { + return nil + } + + if errors.Is(err, context.DeadlineExceeded) { + return &smtp.SMTPError{ + Code: 451, + EnhancedCode: smtp.EnhancedCode{4, 4, 5}, + Message: "High load, try again later", + } + } + + res := &smtp.SMTPError{ + Code: 554, + EnhancedCode: smtp.EnhancedCodeNotSet, + // Err on the side of caution if the error lacks SMTP annotations. If + // we just pass the error text through, we might accidenetally disclose + // details of server configuration. + Message: "Internal server error", + } + + if exterrors.IsTemporary(err) { + res.Code = 451 + } + + ctxInfo := exterrors.Fields(err) + ctxCode, ok := ctxInfo["smtp_code"].(int) + if ok { + res.Code = ctxCode + } + ctxEnchCode, ok := ctxInfo["smtp_enchcode"].(exterrors.EnhancedCode) + if ok { + res.EnhancedCode = smtp.EnhancedCode(ctxEnchCode) + } + ctxMsg, ok := ctxInfo["smtp_msg"].(string) + if ok { + res.Message = ctxMsg + } + + if smtpErr, ok := err.(*smtp.SMTPError); ok { + endp.Log.Printf("plain SMTP error returned, this is deprecated") + res.Code = smtpErr.Code + res.EnhancedCode = smtpErr.EnhancedCode + res.Message = smtpErr.Message + } + + if msgId != "" { + res.Message += " (msg ID = " + msgId + ")" + } + + failedCmds.WithLabelValues(endp.name, command, strconv.Itoa(res.Code), + fmt.Sprintf("%d.%d.%d", + res.EnhancedCode[0], + res.EnhancedCode[1], + res.EnhancedCode[2])).Inc() + + // INTERNATIONALIZATION: See RFC 6531 Section 3.7.4.1. + if mangleUTF8 { + b := strings.Builder{} + b.Grow(len(res.Message)) + for _, ch := range res.Message { + if ch > 128 { + b.WriteRune('?') + } else { + b.WriteRune(ch) + } + } + res.Message = b.String() + } + + return res +} diff --git a/internal/endpoint/smtp/smtp.go b/internal/endpoint/smtp/smtp.go new file mode 100644 index 0000000..8f800e5 --- /dev/null +++ b/internal/endpoint/smtp/smtp.go @@ -0,0 +1,429 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import ( + "bytes" + "context" + "crypto/tls" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + tls2 "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/future" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth" + "github.com/foxcpp/maddy/internal/authz" + "github.com/foxcpp/maddy/internal/limits" + "github.com/foxcpp/maddy/internal/msgpipeline" + "github.com/foxcpp/maddy/internal/proxy_protocol" + "golang.org/x/net/idna" +) + +type Endpoint struct { + saslAuth auth.SASLAuth + serv *smtp.Server + name string + addrs []string + listeners []net.Listener + proxyProtocol *proxy_protocol.ProxyProtocol + pipeline *msgpipeline.MsgPipeline + resolver dns.Resolver + limits *limits.Group + + buffer func(r io.Reader) (buffer.Buffer, error) + + authAlwaysRequired bool + submission bool + lmtp bool + deferServerReject bool + maxLoggedRcptErrors int + maxReceived int + maxHeaderBytes int64 + + sessionCnt atomic.Int32 + + listenersWg sync.WaitGroup + + Log log.Logger +} + +func (endp *Endpoint) Name() string { + return endp.name +} + +func (endp *Endpoint) InstanceName() string { + return endp.name +} + +func New(modName string, addrs []string) (module.Module, error) { + endp := &Endpoint{ + name: modName, + addrs: addrs, + submission: modName == "submission", + lmtp: modName == "lmtp", + resolver: dns.DefaultResolver(), + buffer: buffer.BufferInMemory, + Log: log.Logger{Name: modName}, + saslAuth: auth.SASLAuth{ + Log: log.Logger{Name: modName + "/sasl"}, + }, + } + return endp, nil +} + +func (endp *Endpoint) Init(cfg *config.Map) error { + endp.serv = smtp.NewServer(endp) + endp.serv.ErrorLog = endp.Log + endp.serv.LMTP = endp.lmtp + endp.serv.EnableSMTPUTF8 = true + endp.serv.EnableREQUIRETLS = true + if err := endp.setConfig(cfg); err != nil { + return err + } + + addresses := make([]config.Endpoint, 0, len(endp.addrs)) + for _, addr := range endp.addrs { + saddr, err := config.ParseEndpoint(addr) + if err != nil { + return fmt.Errorf("%s: invalid address: %s", addr, endp.name) + } + + addresses = append(addresses, saddr) + } + + if err := endp.setupListeners(addresses); err != nil { + for _, l := range endp.listeners { + l.Close() + } + return err + } + + allLocal := true + for _, addr := range addresses { + if addr.Scheme != "unix" && !strings.HasPrefix(addr.Host, "127.0.0.") { + allLocal = false + } + } + + if endp.serv.AllowInsecureAuth && !allLocal { + endp.Log.Println("authentication over unencrypted connections is allowed, this is insecure configuration and should be used only for testing!") + } + if endp.serv.TLSConfig == nil { + if !allLocal { + endp.Log.Println("TLS is disabled, this is insecure configuration and should be used only for testing!") + } + + endp.serv.AllowInsecureAuth = true + } + + return nil +} + +func autoBufferMode(maxSize int, dir string) func(io.Reader) (buffer.Buffer, error) { + return func(r io.Reader) (buffer.Buffer, error) { + // First try to read up to N bytes. + initial := make([]byte, maxSize) + actualSize, err := io.ReadFull(r, initial) + if err != nil { + if err == io.ErrUnexpectedEOF { + log.Debugln("autobuffer: keeping the message in RAM (read", actualSize, "bytes, got EOF)") + return buffer.MemoryBuffer{Slice: initial[:actualSize]}, nil + } + if err == io.EOF { + // Special case: message with empty body. + return buffer.MemoryBuffer{}, nil + } + // Some I/O error happened, bail out. + return nil, err + } + if actualSize < maxSize { + // Ok, the message is smaller than N. Make a MemoryBuffer and + // handle it in RAM. + log.Debugln("autobuffer: keeping the message in RAM (read", actualSize, "bytes, got short read)") + return buffer.MemoryBuffer{Slice: initial[:actualSize]}, nil + } + + log.Debugln("autobuffer: spilling the message to the FS") + // The message is big. Dump what we got to the disk and continue writing it there. + return buffer.BufferInFile( + io.MultiReader(bytes.NewReader(initial[:actualSize]), r), + dir) + } +} + +func bufferModeDirective(_ *config.Map, node config.Node) (interface{}, error) { + if len(node.Args) < 1 { + return nil, config.NodeErr(node, "at least one argument required") + } + switch node.Args[0] { + case "ram": + if len(node.Args) > 1 { + return nil, config.NodeErr(node, "no additional arguments for 'ram' mode") + } + return buffer.BufferInMemory, nil + case "fs": + path := filepath.Join(config.StateDirectory, "buffer") + if err := os.MkdirAll(path, 0o700); err != nil { + return nil, err + } + switch len(node.Args) { + case 2: + path = node.Args[1] + fallthrough + case 1: + return func(r io.Reader) (buffer.Buffer, error) { + return buffer.BufferInFile(r, path) + }, nil + default: + return nil, config.NodeErr(node, "too many arguments for 'fs' mode") + } + case "auto": + path := filepath.Join(config.StateDirectory, "buffer") + if err := os.MkdirAll(path, 0o700); err != nil { + return nil, err + } + + maxSize := 1 * 1024 * 1024 // 1 MiB + switch len(node.Args) { + case 3: + path = node.Args[2] + fallthrough + case 2: + var err error + maxSize, err = config.ParseDataSize(node.Args[1]) + if err != nil { + return nil, config.NodeErr(node, "%v", err) + } + fallthrough + case 1: + return autoBufferMode(maxSize, path), nil + default: + return nil, config.NodeErr(node, "too many arguments for 'auto' mode") + } + default: + return nil, config.NodeErr(node, "unknown buffer mode: %v", node.Args[0]) + } +} + +func (endp *Endpoint) setConfig(cfg *config.Map) error { + var ( + hostname string + err error + ioDebug bool + ) + + cfg.Callback("auth", func(m *config.Map, node config.Node) error { + return endp.saslAuth.AddProvider(m, node) + }) + cfg.Bool("sasl_login", false, false, &endp.saslAuth.EnableLogin) + cfg.String("hostname", true, true, "", &hostname) + config.EnumMapped(cfg, "auth_map_normalize", true, false, authz.NormalizeFuncs, authz.NormalizeAuto, + &endp.saslAuth.AuthNormalize) + modconfig.Table(cfg, "auth_map", true, false, nil, &endp.saslAuth.AuthMap) + cfg.Duration("write_timeout", false, false, 1*time.Minute, &endp.serv.WriteTimeout) + cfg.Duration("read_timeout", false, false, 10*time.Minute, &endp.serv.ReadTimeout) + cfg.DataSize("max_message_size", false, false, 32*1024*1024, &endp.serv.MaxMessageBytes) + cfg.DataSize("max_header_size", false, false, 1*1024*1024, &endp.maxHeaderBytes) + cfg.Int("max_recipients", false, false, 20000, &endp.serv.MaxRecipients) + cfg.Int("max_received", false, false, 50, &endp.maxReceived) + cfg.Custom("buffer", false, false, func() (interface{}, error) { + path := filepath.Join(config.StateDirectory, "buffer") + if err := os.MkdirAll(path, 0o700); err != nil { + return nil, err + } + return autoBufferMode(1*1024*1024 /* 1 MiB */, path), nil + }, bufferModeDirective, &endp.buffer) + cfg.Custom("tls", true, endp.name != "lmtp", nil, tls2.TLSDirective, &endp.serv.TLSConfig) + cfg.Custom("proxy_protocol", false, false, nil, proxy_protocol.ProxyProtocolDirective, &endp.proxyProtocol) + cfg.Bool("insecure_auth", endp.name == "lmtp", false, &endp.serv.AllowInsecureAuth) + cfg.Int("smtp_max_line_length", false, false, 4000, &endp.serv.MaxLineLength) + cfg.Bool("io_debug", false, false, &ioDebug) + cfg.Bool("debug", true, false, &endp.Log.Debug) + cfg.Bool("defer_sender_reject", false, true, &endp.deferServerReject) + cfg.Int("max_logged_rcpt_errors", false, false, 5, &endp.maxLoggedRcptErrors) + cfg.Custom("limits", false, false, func() (interface{}, error) { + return &limits.Group{}, nil + }, func(cfg *config.Map, n config.Node) (interface{}, error) { + var g *limits.Group + if err := modconfig.GroupFromNode("limits", n.Args, n, cfg.Globals, &g); err != nil { + return nil, err + } + return g, nil + }, &endp.limits) + cfg.AllowUnknown() + unknown, err := cfg.Process() + if err != nil { + return err + } + + endp.saslAuth.Log.Debug = endp.Log.Debug + + // INTERNATIONALIZATION: See RFC 6531 Section 3.3. + endp.serv.Domain, err = idna.ToASCII(hostname) + if err != nil { + return fmt.Errorf("%s: cannot represent the hostname as an A-label name: %w", endp.name, err) + } + + endp.pipeline, err = msgpipeline.New(cfg.Globals, unknown) + if err != nil { + return err + } + endp.pipeline.Hostname = endp.serv.Domain + endp.pipeline.Resolver = endp.resolver + endp.pipeline.Log = log.Logger{Name: "smtp/pipeline", Debug: endp.Log.Debug} + endp.pipeline.FirstPipeline = true + + if endp.submission { + endp.authAlwaysRequired = true + if len(endp.saslAuth.SASLMechanisms()) == 0 { + return fmt.Errorf("%s: auth. provider must be set for submission endpoint", endp.name) + } + } + + if ioDebug { + endp.serv.Debug = endp.Log.DebugWriter() + endp.Log.Println("I/O debugging is on! It may leak passwords in logs, be careful!") + } + + return nil +} + +func (endp *Endpoint) setupListeners(addresses []config.Endpoint) error { + for _, addr := range addresses { + var l net.Listener + var err error + l, err = net.Listen(addr.Network(), addr.Address()) + if err != nil { + return fmt.Errorf("%s: %w", endp.name, err) + } + endp.Log.Printf("listening on %v", addr) + + if addr.IsTLS() { + if endp.serv.TLSConfig == nil { + return fmt.Errorf("%s: can't bind on SMTPS endpoint without TLS configuration", endp.name) + } + l = tls.NewListener(l, endp.serv.TLSConfig) + } + + if endp.proxyProtocol != nil { + l = proxy_protocol.NewListener(l, endp.proxyProtocol, endp.Log) + } + + endp.listeners = append(endp.listeners, l) + + endp.listenersWg.Add(1) + go func() { + if err := endp.serv.Serve(l); err != nil { + endp.Log.Printf("failed to serve %s: %s", addr, err) + } + endp.listenersWg.Done() + }() + } + + return nil +} + +func (endp *Endpoint) NewSession(conn *smtp.Conn) (smtp.Session, error) { + sess := endp.newSession(conn) + + // Executed before authentication and session initialization. + if err := endp.pipeline.RunEarlyChecks(context.TODO(), &sess.connState); err != nil { + if err := sess.Logout(); err != nil { + endp.Log.Error("early checks logout failed", err) + } + return nil, endp.wrapErr("", true, "EHLO", err) + } + + endp.sessionCnt.Add(1) + + return sess, nil +} + +func (endp *Endpoint) newSession(conn *smtp.Conn) *Session { + s := &Session{ + endp: endp, + log: endp.Log, + sessionCtx: context.Background(), + } + + // Used in tests. + if conn == nil { + return s + } + + s.connState = module.ConnState{ + Hostname: conn.Hostname(), + LocalAddr: conn.Conn().LocalAddr(), + RemoteAddr: conn.Conn().RemoteAddr(), + } + if tlsState, ok := conn.TLSConnectionState(); ok { + s.connState.TLS = tlsState + } + + if endp.serv.LMTP { + s.connState.Proto = "LMTP" + } else { + // Check if TLS connection conn struct is poplated. + // If it is - we are ssing TLS. + if s.connState.TLS.HandshakeComplete { + s.connState.Proto = "ESMTPS" + } else { + s.connState.Proto = "ESMTP" + } + } + + if endp.resolver != nil { + rdnsCtx, cancelRDNS := context.WithCancel(s.sessionCtx) + s.connState.RDNSName = future.New() + s.cancelRDNS = cancelRDNS + go s.fetchRDNSName(rdnsCtx) + } + + return s +} + +func (endp *Endpoint) ConnectionCount() int { + return int(endp.sessionCnt.Load()) +} + +func (endp *Endpoint) Close() error { + endp.serv.Close() + endp.listenersWg.Wait() + return nil +} + +func init() { + module.RegisterEndpoint("smtp", New) + module.RegisterEndpoint("submission", New) + module.RegisterEndpoint("lmtp", New) +} diff --git a/internal/endpoint/smtp/smtp_test.go b/internal/endpoint/smtp/smtp_test.go new file mode 100644 index 0000000..fa46c27 --- /dev/null +++ b/internal/endpoint/smtp/smtp_test.go @@ -0,0 +1,591 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import ( + "flag" + "math/rand" + "net" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/emersion/go-sasl" + "github.com/emersion/go-smtp" + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/auth" + "github.com/foxcpp/maddy/internal/msgpipeline" + "github.com/foxcpp/maddy/internal/testutils" +) + +var testPort string + +const testMsg = "From: \r\n" + + "Subject: Hello there!\r\n" + + "\r\n" + + "foobar\r\n" + +func testEndpoint(t *testing.T, modName string, authMod module.PlainAuth, tgt module.DeliveryTarget, checks []module.Check, cfg []config.Node) *Endpoint { + t.Helper() + + mod, err := New(modName, []string{"tcp://127.0.0.1:" + testPort}) + if err != nil { + t.Fatal(err) + } + endp := mod.(*Endpoint) + + endp.resolver = &mockdns.Resolver{ + Zones: map[string]mockdns.Zone{ + "mx.example.org.": { + A: []string{"127.0.0.1"}, + }, + "1.0.0.127.in-addr.arpa.": { + PTR: []string{"mx.example.org"}, + }, + }, + } + endp.Log = testutils.Logger(t, "smtp") + + cfg = append(cfg, + config.Node{ + Name: "hostname", + Args: []string{"mx.example.com"}, + }, + config.Node{ + Name: "tls", + Args: []string{"off"}, + }, + config.Node{ // To make it succeed, pipeline is actually replaced below. + Name: "deliver_to", + Args: []string{"dummy"}, + }, + ) + + if authMod != nil { + cfg = append(cfg, config.Node{ + Name: "auth", + Args: []string{"dummy"}, + }) + } + + err = endp.Init(config.NewMap(nil, config.Node{ + Children: cfg, + })) + if err != nil { + t.Fatal(err) + } + + endp.saslAuth = auth.SASLAuth{ + Log: testutils.Logger(t, "smtp/saslauth"), + Plain: []module.PlainAuth{authMod}, + } + + endp.pipeline = msgpipeline.Mock(tgt, checks) + endp.pipeline.Hostname = "mx.example.com" + endp.pipeline.Resolver = endp.resolver + endp.pipeline.FirstPipeline = true + endp.pipeline.Log = testutils.Logger(t, "smtp/pipeline") + + return endp +} + +func submitMsg(t *testing.T, cl *smtp.Client, from string, rcpts []string, msg string) error { + return submitMsgOpts(t, cl, from, rcpts, nil, msg) +} + +func submitMsgOpts(t *testing.T, cl *smtp.Client, from string, rcpts []string, opts *smtp.MailOptions, msg string) error { + t.Helper() + + // Error for this one is ignored because it fails if EHLO was already sent + // and submitMsg can happen multiple times. + _ = cl.Hello("mx.example.org") + if err := cl.Mail(from, opts); err != nil { + return err + } + for _, rcpt := range rcpts { + if err := cl.Rcpt(rcpt, &smtp.RcptOptions{}); err != nil { + return err + } + } + data, err := cl.Data() + if err != nil { + return err + } + if _, err := data.Write([]byte(msg)); err != nil { + return err + } + + return data.Close() +} + +func TestSMTPDelivery(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsg(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + msgID := testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, "") + + receivedPrefix := `from mx.example.org (mx.example.org [127.0.0.1]) by mx.example.com (envelope-sender ) with ESMTP id ` + msgID + + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } + + if msg.MsgMeta.Conn.Proto != "ESMTP" { + t.Error("Wrong SrcProto:", msg.MsgMeta.Conn.Proto) + } + + rdnsName, _ := msg.MsgMeta.Conn.RDNSName.Get() + if rdnsName, _ := rdnsName.(string); rdnsName != "mx.example.org" { + t.Error("Wrong rDNS name:", rdnsName) + } +} + +func TestSMTPDelivery_rDNSError(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + + endp.resolver.(*mockdns.Resolver).Zones["1.0.0.127.in-addr.arpa."] = mockdns.Zone{ + Err: &net.DNSError{ + Name: "1.0.0.127.in-addr.arpa.", + Server: "127.0.0.1:53", + Err: "bad", + IsNotFound: false, + }, + } + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsg(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, "") + + rdnsName, err := msg.MsgMeta.Conn.RDNSName.Get() + if rdnsName != nil || err == nil { + t.Errorf("Wrong rDNS result: %#+v (%v)", rdnsName, err) + } +} + +func TestSMTPDelivery_EarlyCheck_Fail(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, []module.Check{ + &testutils.Check{ + EarlyErr: &exterrors.SMTPError{ + Code: 523, + Message: "Hey", + }, + }, + }, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = cl.Mail("sender@example.org", nil) + if err == nil { + t.Fatal("Expected an error, got none") + } + + smtpErr, ok := err.(*smtp.SMTPError) + if !ok { + t.Fatal("Non-SMTPError returned") + } + + if smtpErr.Code != 523 { + t.Fatal("Wrong SMTP code:", smtpErr.Code) + } + if smtpErr.Message != "Hey" { + t.Fatal("Wrong SMTP message:", smtpErr.Message) + } +} + +func TestSMTPDeliver_CheckError(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, []module.Check{ + &testutils.Check{ + ConnRes: module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 523, + Message: "Hey", + }, + Reject: true, + }, + }, + }, nil) + endp.deferServerReject = false + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = cl.Mail("sender@example.org", nil) + if err == nil { + t.Fatal("Expected an error, got none") + } + smtpErr, ok := err.(*smtp.SMTPError) + if !ok { + t.Fatal("Non-SMTPError returned") + } + + if smtpErr.Code != 523 { + t.Fatal("Wrong SMTP code:", smtpErr.Code) + } + if !strings.HasPrefix(smtpErr.Message, "Hey") { + t.Fatal("Wrong SMTP message:", smtpErr.Message) + } +} + +func TestSMTPDeliver_CheckError_Deferred(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, []module.Check{ + &testutils.Check{ + ConnRes: module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 523, + Message: "Hey", + }, + Reject: true, + }, + }, + }, nil) + endp.deferServerReject = true + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = cl.Mail("sender@example.org", nil) + if err != nil { + t.Fatal(err) + } + + checkErr := func(err error) { + if err == nil { + t.Fatal("Expected an error, got none") + } + smtpErr, ok := err.(*smtp.SMTPError) + if !ok { + t.Error("Non-SMTPError returned") + return + } + + if smtpErr.Code != 523 { + t.Error("Wrong SMTP code:", smtpErr.Code) + } + if !strings.HasPrefix(smtpErr.Message, "Hey") { + t.Error("Wrong SMTP message:", smtpErr.Message) + } + } + + checkErr(cl.Rcpt("test1@example.org", &smtp.RcptOptions{})) + checkErr(cl.Rcpt("test1@example.org", &smtp.RcptOptions{})) + checkErr(cl.Rcpt("test2@example.org", &smtp.RcptOptions{})) +} + +func TestSMTPDelivery_Multi(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsg(t, cl, "sender1@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) + if err != nil { + t.Fatal(err) + } + err = submitMsg(t, cl, "sender2@example.org", []string{"rcpt3@example.com", "rcpt4@example.com"}, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 2 { + t.Fatal("Expected two messages, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + msgID := testutils.CheckMsgID(t, &msg, "sender1@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, "") + receivedPrefix := `from mx.example.org (mx.example.org [127.0.0.1]) by mx.example.com (envelope-sender ) with ESMTP id ` + msgID + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } + + msg = tgt.Messages[1] + msgID = testutils.CheckMsgID(t, &msg, "sender2@example.org", []string{"rcpt3@example.com", "rcpt4@example.com"}, "") + receivedPrefix = `from mx.example.org (mx.example.org [127.0.0.1]) by mx.example.com (envelope-sender ) with ESMTP id ` + msgID + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } +} + +func TestSMTPDelivery_AbortData(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Hello("mx.example.org"); err != nil { + t.Fatal(err) + } + if err := cl.Mail("sender@example.org", nil); err != nil { + t.Fatal(err) + } + if err := cl.Rcpt("test@example.com", &smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + data, err := cl.Data() + if err != nil { + t.Fatal(err) + } + if _, err := data.Write([]byte(testMsg)); err != nil { + t.Fatal(err) + } + + // Then.. Suddenly, close the connection without sending the final dot. + cl.Close() + + time.Sleep(250 * time.Millisecond) + + if len(tgt.Messages) != 0 { + t.Fatal("Expected no messages, got", len(tgt.Messages)) + } +} + +func TestSMTPDelivery_EmptyMessage(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Hello("mx.example.org"); err != nil { + t.Fatal(err) + } + if err := cl.Mail("sender@example.org", nil); err != nil { + t.Fatal(err) + } + if err := cl.Rcpt("test@example.com", &smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + data, err := cl.Data() + if err != nil { + t.Fatal(err) + } + if err := data.Close(); err != nil { + t.Fatal(err) + } + + time.Sleep(250 * time.Millisecond) + + if len(tgt.Messages) != 1 { + t.Fatal("Expected 1 message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + if len(msg.Body) != 0 { + t.Fatal("Expected an empty body, got", len(msg.Body)) + } +} + +func TestSMTPDelivery_AbortLogout(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Hello("mx.example.org"); err != nil { + t.Fatal(err) + } + if err := cl.Mail("sender@example.org", nil); err != nil { + t.Fatal(err) + } + if err := cl.Rcpt("test@example.com", &smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + // Then.. Suddenly, close the connection. + cl.Close() + + time.Sleep(250 * time.Millisecond) + + if len(tgt.Messages) != 0 { + t.Fatal("Expected no messages, got", len(tgt.Messages)) + } +} + +func TestSMTPDelivery_Reset(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Mail("from-garbage@example.org", nil); err != nil { + t.Fatal(err) + } + if err := cl.Rcpt("to-garbage@example.org", &smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + if err := cl.Reset(); err != nil { + t.Fatal(err) + } + + // then submit the message as if nothing happened. + + err = submitMsg(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, "") +} + +func TestSMTPDelivery_SubmissionAuthRequire(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "submission", &module.Dummy{}, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Mail("from-garbage@example.org", nil); err == nil { + t.Fatal("Expected an error, got none") + } +} + +func TestSMTPDelivery_SubmissionAuthOK(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "submission", &module.Dummy{}, &tgt, nil, nil) + defer endp.Close() + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Auth(sasl.NewPlainClient("", "user", "password")); err != nil { + t.Fatal(err) + } + + if err := submitMsg(t, cl, "sender@example.org", []string{"rcpt@example.org"}, testMsg); err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + msgID := testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt@example.org"}, "") + + if msg.MsgMeta.Conn.AuthUser != "user" { + t.Error("Wrong AuthUser:", msg.MsgMeta.Conn.AuthUser) + } + if msg.MsgMeta.Conn.AuthPassword != "password" { + t.Error("Wrong AuthPassword:", msg.MsgMeta.Conn.AuthPassword) + } + + receivedPrefix := `by mx.example.com (envelope-sender ) with ESMTP id ` + msgID + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } + + if msg.Header.Get("Message-ID") == "" { + t.Error("No submissionPrepare run") + } +} + +func TestMain(m *testing.M) { + remoteSmtpPort := flag.String("test.smtpport", "random", "(maddy) SMTP port to use for connections in tests") + flag.Parse() + + if *remoteSmtpPort == "random" { + *remoteSmtpPort = strconv.Itoa(rand.Intn(65536-10000) + 10000) + } + + testPort = *remoteSmtpPort + os.Exit(m.Run()) +} diff --git a/internal/endpoint/smtp/smtputf8_test.go b/internal/endpoint/smtp/smtputf8_test.go new file mode 100644 index 0000000..b3f5701 --- /dev/null +++ b/internal/endpoint/smtp/smtputf8_test.go @@ -0,0 +1,361 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import ( + "strings" + "testing" + + "github.com/emersion/go-smtp" + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestSMTPUTF8_MangleStatusMessage(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, []module.Check{ + &testutils.Check{ + ConnRes: module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 523, + Message: "Hey 凱凱", + }, + Reject: true, + }, + }, + }, nil) + endp.deferServerReject = false + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = cl.Mail("sender@example.org", nil) + if err == nil { + t.Fatal("Expected an error, got none") + } + smtpErr, ok := err.(*smtp.SMTPError) + if !ok { + t.Fatal("Non-SMTPError returned") + } + + if smtpErr.Code != 523 { + t.Fatal("Wrong SMTP code:", smtpErr.Code) + } + if !strings.HasPrefix(smtpErr.Message, "Hey ??") { + t.Fatal("Wrong SMTP message:", smtpErr.Message) + } +} + +func TestSMTP_RejectNonASCIIFrom(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + endp.deferServerReject = false + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsg(t, cl, "ѣ@example.org", []string{"rcpt@example.com"}, testMsg) + + smtpErr, ok := err.(*smtp.SMTPError) + if !ok { + t.Fatal("Non-SMTPError returned") + } + if smtpErr.Code != 550 { + t.Fatal("Wrong SMTP code:", smtpErr.Code) + } + if smtpErr.EnhancedCode != (smtp.EnhancedCode{5, 6, 7}) { + t.Fatal("Wrong SMTP ench. code:", smtpErr.EnhancedCode) + } +} + +func TestSMTPUTF8_NormalizeCaseFoldFrom(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + endp.deferServerReject = false + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsgOpts(t, cl, "foo@E\u0301.example.org", []string{"rcpt@example.com"}, &smtp.MailOptions{ + UTF8: true, + }, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + testutils.CheckMsgID(t, &msg, "foo@é.example.org", []string{"rcpt@example.com"}, "") +} + +func TestSMTP_RejectNonASCIIRcpt(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + endp.deferServerReject = false + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsg(t, cl, "x@example.org", []string{"ѣ@example.org"}, testMsg) + + smtpErr, ok := err.(*smtp.SMTPError) + if !ok { + t.Fatal("Non-SMTPError returned") + } + if smtpErr.Code != 553 { + t.Fatal("Wrong SMTP code:", smtpErr.Code) + } + if smtpErr.EnhancedCode != (smtp.EnhancedCode{5, 6, 7}) { + t.Fatal("Wrong SMTP ench. code:", smtpErr.EnhancedCode) + } +} + +func TestSMTPUTF8_NormalizeCaseFoldRcpt(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + endp.deferServerReject = false + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsgOpts(t, cl, "x@example.org", []string{"foo@E\u0301.example.org"}, &smtp.MailOptions{ + UTF8: true, + }, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + testutils.CheckMsgID(t, &msg, "x@example.org", []string{"foo@é.example.org"}, "") +} + +func TestSMTPUTF8_NoMangleStatusMessage(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, []module.Check{ + &testutils.Check{ + ConnRes: module.CheckResult{ + Reason: &exterrors.SMTPError{ + Code: 523, + Message: "Hey 凱凱", + }, + Reject: true, + }, + }, + }, nil) + endp.deferServerReject = false + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = cl.Mail("sender@example.org", &smtp.MailOptions{ + UTF8: true, + }) + if err == nil { + t.Fatal("Expected an error, got none") + } + smtpErr, ok := err.(*smtp.SMTPError) + if !ok { + t.Fatal("Non-SMTPError returned") + } + + if smtpErr.Code != 523 { + t.Fatal("Wrong SMTP code:", smtpErr.Code) + } + if !strings.HasPrefix(smtpErr.Message, "Hey 凱凱") { + t.Fatal("Wrong SMTP message:", smtpErr.Message) + } +} + +func TestSMTPUTF8_Received_EHLO_ALabel(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Hello("凱凱.invalid"); err != nil { + t.Fatal(err) + } + + err = submitMsg(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + msgID := testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, "") + + receivedPrefix := `from xn--y9qa.invalid (mx.example.org [127.0.0.1]) by mx.example.com (envelope-sender ) with ESMTP id ` + msgID + + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } +} + +func TestSMTPUTF8_Received_rDNS_ALabel(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + endp.resolver.(*mockdns.Resolver).Zones["1.0.0.127.in-addr.arpa."] = mockdns.Zone{ + PTR: []string{"凱凱.invalid."}, + } + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsg(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + msgID := testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, "") + + receivedPrefix := `from mx.example.org (xn--y9qa.invalid [127.0.0.1]) by mx.example.com (envelope-sender ) with ESMTP id ` + msgID + + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } +} + +func TestSMTPUTF8_Received_rDNS_ULabel(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + endp.resolver.(*mockdns.Resolver).Zones["1.0.0.127.in-addr.arpa."] = mockdns.Zone{ + PTR: []string{"凱凱.invalid."}, + } + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + err = submitMsgOpts(t, cl, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, &smtp.MailOptions{ + UTF8: true, + }, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + msgID := testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}, "") + + receivedPrefix := `from mx.example.org (凱凱.invalid [127.0.0.1]) by mx.example.com (envelope-sender ) with UTF8ESMTP id ` + msgID + + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } +} + +func TestSMTPUTF8_Received_EHLO_ULabel(t *testing.T) { + tgt := testutils.Target{} + endp := testEndpoint(t, "smtp", nil, &tgt, nil, nil) + defer endp.Close() + defer testutils.WaitForConnsClose(t, endp.serv) + + cl, err := smtp.Dial("127.0.0.1:" + testPort) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + + if err := cl.Hello("凱凱.invalid"); err != nil { + t.Fatal(err) + } + + err = submitMsgOpts(t, cl, "sender@example.org", []string{"rcpt@example.com"}, &smtp.MailOptions{ + UTF8: true, + }, testMsg) + if err != nil { + t.Fatal(err) + } + + if len(tgt.Messages) != 1 { + t.Fatal("Expected a message, got", len(tgt.Messages)) + } + msg := tgt.Messages[0] + msgID := testutils.CheckMsgID(t, &msg, "sender@example.org", []string{"rcpt@example.com"}, "") + + // Also, 'with UTF8ESMTP'. + receivedPrefix := `from 凱凱.invalid (mx.example.org [127.0.0.1]) by mx.example.com (envelope-sender ) with UTF8ESMTP id ` + msgID + + if !strings.HasPrefix(msg.Header.Get("Received"), receivedPrefix) { + t.Error("Wrong Received contents:", msg.Header.Get("Received")) + } +} diff --git a/internal/endpoint/smtp/submission.go b/internal/endpoint/smtp/submission.go new file mode 100644 index 0000000..316c8b6 --- /dev/null +++ b/internal/endpoint/smtp/submission.go @@ -0,0 +1,148 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import ( + "errors" + "fmt" + "net/mail" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/google/uuid" +) + +var ( + msgIDField = func() (string, error) { + id, err := uuid.NewRandom() + if err != nil { + return "", err + } + return id.String(), nil + } + + now = time.Now +) + +func (s *Session) submissionPrepare(msgMeta *module.MsgMetadata, header *textproto.Header) error { + msgMeta.DontTraceSender = true + + if header.Get("Message-ID") == "" { + msgId, err := msgIDField() + if err != nil { + return errors.New("Message-ID generation failed") + } + s.log.Msg("adding missing Message-ID") + header.Set("Message-ID", "<"+msgId+"@"+s.endp.serv.Domain+">") + } + + if header.Get("From") == "" { + return &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 6, 0}, + Message: "Message does not contains a From header field", + Misc: map[string]interface{}{ + "modifier": "submission_prepare", + }, + } + } + + for _, hdr := range [...]string{"Sender"} { + if value := header.Get(hdr); value != "" { + if _, err := mail.ParseAddress(value); err != nil { + return &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 6, 0}, + Message: fmt.Sprintf("Invalid address in %s", hdr), + Misc: map[string]interface{}{ + "modifier": "submission_prepare", + "addr": value, + }, + Err: err, + } + } + } + } + for _, hdr := range [...]string{"To", "Cc", "Bcc", "Reply-To"} { + if value := header.Get(hdr); value != "" { + if _, err := mail.ParseAddressList(value); err != nil { + return &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 6, 0}, + Message: fmt.Sprintf("Invalid address in %s", hdr), + Misc: map[string]interface{}{ + "modifier": "submission_prepare", + "addr": value, + }, + Err: err, + } + } + } + } + + addrs, err := mail.ParseAddressList(header.Get("From")) + if err != nil { + return &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 6, 0}, + Message: "Invalid address in From", + Misc: map[string]interface{}{ + "modifier": "submission_prepare", + "addr": header.Get("From"), + }, + Err: err, + } + } + + // https://tools.ietf.org/html/rfc5322#section-3.6.2 + // If From contains multiple addresses, Sender field must be present. + if len(addrs) > 1 && header.Get("Sender") == "" { + return &exterrors.SMTPError{ + Code: 554, + EnhancedCode: exterrors.EnhancedCode{5, 6, 0}, + Message: "Missing Sender header field", + Misc: map[string]interface{}{ + "modifier": "submission_prepare", + "from": header.Get("From"), + }, + } + } + + if dateHdr := header.Get("Date"); dateHdr != "" { + _, err := parseMessageDateTime(dateHdr) + if err != nil { + return &exterrors.SMTPError{ + Code: 554, + Message: "Malformed Date header", + Misc: map[string]interface{}{ + "modifier": "submission_prepare", + "date": dateHdr, + }, + Err: err, + } + } + } else { + s.log.Msg("adding missing Date header") + header.Set("Date", now().UTC().Format("Mon, 2 Jan 2006 15:04:05 -0700")) + } + + return nil +} diff --git a/internal/endpoint/smtp/submission_test.go b/internal/endpoint/smtp/submission_test.go new file mode 100644 index 0000000..f257364 --- /dev/null +++ b/internal/endpoint/smtp/submission_test.go @@ -0,0 +1,165 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp + +import ( + "reflect" + "testing" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/module" +) + +func init() { + msgIDField = func() (string, error) { + return "A", nil + } + + now = func() time.Time { + return time.Unix(0, 0) + } +} + +func TestSubmissionPrepare(t *testing.T) { + test := func(hdrMap, expectedMap map[string][]string) { + t.Helper() + + hdr := textproto.Header{} + for k, v := range hdrMap { + for _, field := range v { + hdr.Add(k, field) + } + } + + endp := testEndpoint(t, "submission", &module.Dummy{}, &module.Dummy{}, nil, nil) + defer func() { + // Synchronize the endpoint initialization. + // Otherwise Close will race with Serve called by setupListeners. + cl, _ := smtp.Dial("127.0.0.1:" + testPort) + cl.Close() + + endp.Close() + }() + + session, err := endp.NewSession(nil) + if err != nil { + t.Fatal(err) + } + + err = session.(*Session).submissionPrepare(&module.MsgMetadata{}, &hdr) + if expectedMap == nil { + if err == nil { + t.Error("Expected an error, got none") + } + t.Log(err) + return + } + if expectedMap != nil && err != nil { + t.Error("Unexpected error:", err) + return + } + + resMap := make(map[string][]string) + for field := hdr.Fields(); field.Next(); { + resMap[field.Key()] = append(resMap[field.Key()], field.Value()) + } + + if !reflect.DeepEqual(expectedMap, resMap) { + t.Errorf("wrong header result\nwant %#+v\ngot %#+v", expectedMap, resMap) + } + } + + // No From field. + test(map[string][]string{}, nil) + + // Malformed From field. + test(map[string][]string{ + "From": {", \"\""}, + }, nil) + test(map[string][]string{ + "From": {" adasda"}, + }, nil) + + // Malformed Reply-To. + test(map[string][]string{ + "From": {""}, + "Reply-To": {", \"\""}, + }, nil) + + // Malformed CC. + test(map[string][]string{ + "From": {""}, + "Reply-To": {""}, + "Cc": {", \"\""}, + }, nil) + + // Malformed Sender. + test(map[string][]string{ + "From": {""}, + "Reply-To": {""}, + "Cc": {""}, + "Sender": {" asd"}, + }, nil) + + // Multiple From + no Sender. + test(map[string][]string{ + "From": {", "}, + }, nil) + + // Multiple From + valid Sender. + test(map[string][]string{ + "From": {", "}, + "Sender": {""}, + "Date": {"Fri, 22 Nov 2019 20:51:31 +0800"}, + "Message-Id": {""}, + }, map[string][]string{ + "From": {", "}, + "Sender": {""}, + "Date": {"Fri, 22 Nov 2019 20:51:31 +0800"}, + "Message-Id": {""}, + }) + + // Add missing Message-Id. + test(map[string][]string{ + "From": {""}, + "Date": {"Fri, 22 Nov 2019 20:51:31 +0800"}, + }, map[string][]string{ + "From": {""}, + "Date": {"Fri, 22 Nov 2019 20:51:31 +0800"}, + "Message-Id": {""}, + }) + + // Malformed Date. + test(map[string][]string{ + "From": {""}, + "Date": {"not a date"}, + }, nil) + + // Add missing Date. + test(map[string][]string{ + "From": {""}, + "Message-Id": {""}, + }, map[string][]string{ + "From": {""}, + "Message-Id": {""}, + "Date": {"Thu, 1 Jan 1970 00:00:00 +0000"}, + }) +} diff --git a/internal/imap_filter/command/command.go b/internal/imap_filter/command/command.go new file mode 100644 index 0000000..58146dc --- /dev/null +++ b/internal/imap_filter/command/command.go @@ -0,0 +1,207 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package command + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "regexp" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +const modName = "imap.filter.command" + +var placeholderRe = regexp.MustCompile(`{[a-zA-Z0-9_]+?}`) + +type Check struct { + instName string + log log.Logger + + cmd string + cmdArgs []string +} + +func (c *Check) IMAPFilter(accountName string, rcptTo string, msgMeta *module.MsgMetadata, hdr textproto.Header, body buffer.Buffer) (folder string, flags []string, err error) { + cmd, args := c.expandCommand(msgMeta, accountName, rcptTo, hdr) + + var buf bytes.Buffer + _ = textproto.WriteHeader(&buf, hdr) + bR, err := body.Open() + if err != nil { + return "", nil, err + } + + return c.run(cmd, args, io.MultiReader(bytes.NewReader(buf.Bytes()), bR)) +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + c := &Check{ + instName: instName, + log: log.Logger{Name: modName, Debug: log.DefaultLogger.Debug}, + } + + if len(inlineArgs) == 0 { + return nil, errors.New("command: at least one argument is required (command name)") + } + + c.cmd = inlineArgs[0] + c.cmdArgs = inlineArgs[1:] + + return c, nil +} + +func (c *Check) Name() string { + return modName +} + +func (c *Check) InstanceName() string { + return c.instName +} + +func (c *Check) Init(cfg *config.Map) error { + // Check whether the inline argument command is usable. + if _, err := exec.LookPath(c.cmd); err != nil { + return fmt.Errorf("command: %w", err) + } + + _, err := cfg.Process() + return err +} + +func (c *Check) expandCommand(msgMeta *module.MsgMetadata, accountName string, rcptTo string, hdr textproto.Header) (string, []string) { + expArgs := make([]string, len(c.cmdArgs)) + + for i, arg := range c.cmdArgs { + expArgs[i] = placeholderRe.ReplaceAllStringFunc(arg, func(placeholder string) string { + switch placeholder { + case "{auth_user}": + if msgMeta.Conn == nil { + return "" + } + return msgMeta.Conn.AuthUser + case "{source_ip}": + if msgMeta.Conn == nil { + return "" + } + tcpAddr, _ := msgMeta.Conn.RemoteAddr.(*net.TCPAddr) + if tcpAddr == nil { + return "" + } + return tcpAddr.IP.String() + case "{source_host}": + if msgMeta.Conn == nil { + return "" + } + return msgMeta.Conn.Hostname + case "{source_rdns}": + if msgMeta.Conn == nil { + return "" + } + valI, err := msgMeta.Conn.RDNSName.Get() + if err != nil { + return "" + } + if valI == nil { + return "" + } + return valI.(string) + case "{msg_id}": + return msgMeta.ID + case "{sender}": + return msgMeta.OriginalFrom + case "{rcpt_to}": + return rcptTo + case "{original_rcpt_to}": + oldestOriginalRcpt := rcptTo + for originalRcpt, ok := rcptTo, true; ok; originalRcpt, ok = msgMeta.OriginalRcpts[originalRcpt] { + oldestOriginalRcpt = originalRcpt + } + return oldestOriginalRcpt + case "{subject}": + return hdr.Get("Subject") + case "{account_name}": + return accountName + } + return placeholder + }) + } + + return c.cmd, expArgs +} + +func (c *Check) run(cmdName string, args []string, stdin io.Reader) (string, []string, error) { + c.log.Debugln("running", cmdName, args) + + cmd := exec.Command(cmdName, args...) + cmd.Stdin = stdin + stdout, err := cmd.StdoutPipe() + if err != nil { + return "", nil, err + } + + if err := cmd.Start(); err != nil { + return "", nil, err + } + + scnr := bufio.NewScanner(stdout) + var ( + folder string + flags []string + ) + if scnr.Scan() { + folder = scnr.Text() + } + for scnr.Scan() { + flags = append(flags, scnr.Text()) + } + if err := scnr.Err(); err != nil { + return "", nil, err + } + + err = cmd.Wait() + if err != nil { + if _, ok := err.(*exec.ExitError); !ok { + // If that's not ExitError, the process may still be running. We do + // not want this. + if err := cmd.Process.Signal(os.Interrupt); err != nil { + c.log.Error("failed to kill process", err) + } + } + return "", nil, err + } + + c.log.Debugf("folder: %s, extra flags: %v", folder, flags) + + return folder, flags, nil +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/imap_filter/group.go b/internal/imap_filter/group.go new file mode 100644 index 0000000..c1c5ecc --- /dev/null +++ b/internal/imap_filter/group.go @@ -0,0 +1,92 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imap_filter + +import ( + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +// Group wraps multiple modifiers and runs them serially. +// +// It is also registered as a module under 'modifiers' name and acts as a +// module group. +type Group struct { + instName string + Filters []module.IMAPFilter + log log.Logger +} + +func NewGroup(_, instName string, _, _ []string) (module.Module, error) { + return &Group{ + instName: instName, + log: log.Logger{Name: "imap_filters", Debug: log.DefaultLogger.Debug}, + }, nil +} + +func (g *Group) IMAPFilter(accountName string, rcptTo string, meta *module.MsgMetadata, hdr textproto.Header, body buffer.Buffer) (folder string, flags []string, err error) { + if g == nil { + return "", nil, nil + } + var ( + finalFolder string + finalFlags = make([]string, 0, len(g.Filters)) + ) + for _, f := range g.Filters { + folder, flags, err := f.IMAPFilter(accountName, rcptTo, meta, hdr, body) + if err != nil { + g.log.Error("IMAP filter failed", err) + continue + } + if folder != "" && finalFolder == "" { + finalFolder = folder + } + finalFlags = append(finalFlags, flags...) + } + return finalFolder, finalFlags, nil +} + +func (g *Group) Init(cfg *config.Map) error { + for _, node := range cfg.Block.Children { + mod, err := modconfig.IMAPFilter(cfg.Globals, append([]string{node.Name}, node.Args...), node) + if err != nil { + return err + } + + g.Filters = append(g.Filters, mod) + } + + return nil +} + +func (g *Group) Name() string { + return "modifiers" +} + +func (g *Group) InstanceName() string { + return g.instName +} + +func init() { + module.Register("imap_filters", NewGroup) +} diff --git a/internal/libdns/acmedns.go b/internal/libdns/acmedns.go new file mode 100644 index 0000000..cb657ee --- /dev/null +++ b/internal/libdns/acmedns.go @@ -0,0 +1,28 @@ +//go:build libdns_acmedns || libdns_all +// +build libdns_acmedns libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/acmedns" +) + +func init() { + module.Register("libdns.acmedns", func(modName, instName string, _, _ []string) (module.Module, error) { + p := acmedns.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("username", false, true, "", &p.Username) + c.String("password", false, true, "", &p.Password) + c.String("subdomain", false, true, "", &p.Subdomain) + c.String("server_url", false, true, "", &p.ServerURL) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/alidns.go b/internal/libdns/alidns.go new file mode 100644 index 0000000..cb980be --- /dev/null +++ b/internal/libdns/alidns.go @@ -0,0 +1,26 @@ +//go:build libdns_alidns || libdns_all +// +build libdns_alidns libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/alidns" +) + +func init() { + module.Register("libdns.alidns", func(modName, instName string, _, _ []string) (module.Module, error) { + p := alidns.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("key_id", false, false, "", &p.AccKeyID) + c.String("key_secret", false, false, "", &p.AccKeySecret) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/cloudflare.go b/internal/libdns/cloudflare.go new file mode 100644 index 0000000..3031953 --- /dev/null +++ b/internal/libdns/cloudflare.go @@ -0,0 +1,25 @@ +//go:build libdns_cloudflare || !libdns_separate +// +build libdns_cloudflare !libdns_separate + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/cloudflare" +) + +func init() { + module.Register("libdns.cloudflare", func(modName, instName string, _, _ []string) (module.Module, error) { + p := cloudflare.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_token", false, false, "", &p.APIToken) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/digitalocean.go b/internal/libdns/digitalocean.go new file mode 100644 index 0000000..98b77b0 --- /dev/null +++ b/internal/libdns/digitalocean.go @@ -0,0 +1,25 @@ +//go:build libdns_digitalocean || !libdns_separate +// +build libdns_digitalocean !libdns_separate + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/digitalocean" +) + +func init() { + module.Register("libdns.digitalocean", func(modName, instName string, _, _ []string) (module.Module, error) { + p := digitalocean.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_token", false, false, "", &p.APIToken) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/gandi.go b/internal/libdns/gandi.go new file mode 100644 index 0000000..62c7c2d --- /dev/null +++ b/internal/libdns/gandi.go @@ -0,0 +1,38 @@ +//go:build libdns_gandi || !libdns_separate +// +build libdns_gandi !libdns_separate + +package libdns + +import ( + "fmt" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/gandi" +) + +func init() { + module.Register("libdns.gandi", func(modName, instName string, _, _ []string) (module.Module, error) { + p := gandi.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_token", false, false, "", &p.APIToken) + c.String("personal_token", false, false, "", &p.BearerToken) + }, + afterConfig: func() error { + if p.APIToken != "" { + log.Println("libdns.gandi: api_token is deprecated, use personal_token instead (https://api.gandi.net/docs/authentication/)") + } + if p.APIToken == "" && p.BearerToken == "" { + return fmt.Errorf("libdns.gandi: either api_token or personal_token should be specified") + } + return nil + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/gcore.go b/internal/libdns/gcore.go new file mode 100644 index 0000000..01d7afb --- /dev/null +++ b/internal/libdns/gcore.go @@ -0,0 +1,34 @@ +//go:build libdns_gcore || !libdns_separate +// +build libdns_gcore !libdns_separate + +package libdns + +import ( + "fmt" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/gcore" +) + +func init() { + module.Register("libdns.gcore", func(modName, instName string, _, _ []string) (module.Module, error) { + p := gcore.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_key", false, false, "", &p.APIKey) + }, + afterConfig: func() error { + if p.APIKey == "" { + return fmt.Errorf("libdns.gcore: api_key should be specified") + } + return nil + }, + + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/googleclouddns.go b/internal/libdns/googleclouddns.go new file mode 100644 index 0000000..b59c056 --- /dev/null +++ b/internal/libdns/googleclouddns.go @@ -0,0 +1,26 @@ +//go:build libdns_googleclouddns || libdns_all +// +build libdns_googleclouddns libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/googleclouddns" +) + +func init() { + module.Register("libdns.googleclouddns", func(modName, instName string, _, _ []string) (module.Module, error) { + p := googleclouddns.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("project", false, true, "", &p.Project) + c.String("service_account_json", false, false, "", &p.ServiceAccountJSON) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/hetzner.go b/internal/libdns/hetzner.go new file mode 100644 index 0000000..b360641 --- /dev/null +++ b/internal/libdns/hetzner.go @@ -0,0 +1,25 @@ +//go:build libdns_hetzner || !libdns_separate +// +build libdns_hetzner !libdns_separate + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/hetzner" +) + +func init() { + module.Register("libdns.hetzner", func(modName, instName string, _, _ []string) (module.Module, error) { + p := hetzner.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_token", false, false, "", &p.AuthAPIToken) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/leaseweb.go b/internal/libdns/leaseweb.go new file mode 100644 index 0000000..23af12d --- /dev/null +++ b/internal/libdns/leaseweb.go @@ -0,0 +1,25 @@ +//go:build libdns_leaseweb || libdns_all +// +build libdns_leaseweb libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/leaseweb" +) + +func init() { + module.Register("libdns.leaseweb", func(modName, instName string, _, _ []string) (module.Module, error) { + p := leaseweb.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_key", false, false, "", &p.APIKey) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/metaname.go b/internal/libdns/metaname.go new file mode 100644 index 0000000..2e37ddd --- /dev/null +++ b/internal/libdns/metaname.go @@ -0,0 +1,28 @@ +//go:build libdns_metaname || libdns_all +// +build libdns_metaname libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/metaname" +) + +func init() { + module.Register("libdns.metaname", func(modName, instName string, _, _ []string) (module.Module, error) { + p := metaname.Provider{ + Endpoint: "https://metaname.net/api/1.1", + } + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_key", false, false, "", &p.APIKey) + c.String("account_ref", false, false, "", &p.AccountReference) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/namecheap.go b/internal/libdns/namecheap.go new file mode 100644 index 0000000..656ebe5 --- /dev/null +++ b/internal/libdns/namecheap.go @@ -0,0 +1,28 @@ +//go:build go1.16 +// +build go1.16 + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/namecheap" +) + +func init() { + module.Register("libdns.namecheap", func(modName, instName string, _, _ []string) (module.Module, error) { + p := namecheap.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_key", false, true, "", &p.APIKey) + c.String("api_username", false, true, "", &p.User) + c.String("endpoint", false, false, "", &p.APIEndpoint) + c.String("client_ip", false, false, "", &p.ClientIP) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/namedotcom.go b/internal/libdns/namedotcom.go new file mode 100644 index 0000000..3ce3c51 --- /dev/null +++ b/internal/libdns/namedotcom.go @@ -0,0 +1,28 @@ +//go:build libdns_namedotdom || libdns_all +// +build libdns_namedotdom libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/namedotcom" +) + +func init() { + module.Register("libdns.namedotcom", func(modName, instName string, _, _ []string) (module.Module, error) { + p := namedotcom.Provider{ + Server: "https://api.name.com", + } + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("user", false, false, "", &p.User) + c.String("token", false, false, "", &p.Token) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/provider_module.go b/internal/libdns/provider_module.go new file mode 100644 index 0000000..7556150 --- /dev/null +++ b/internal/libdns/provider_module.go @@ -0,0 +1,35 @@ +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/libdns/libdns" +) + +type ProviderModule struct { + libdns.RecordDeleter + libdns.RecordAppender + setConfig func(c *config.Map) + afterConfig func() error + + instName string + modName string +} + +func (p *ProviderModule) Init(cfg *config.Map) error { + p.setConfig(cfg) + _, err := cfg.Process() + if p.afterConfig != nil { + if err := p.afterConfig(); err != nil { + return err + } + } + return err +} + +func (p *ProviderModule) Name() string { + return p.modName +} + +func (p *ProviderModule) InstanceName() string { + return p.instName +} diff --git a/internal/libdns/rfc2136.go b/internal/libdns/rfc2136.go new file mode 100644 index 0000000..19751f6 --- /dev/null +++ b/internal/libdns/rfc2136.go @@ -0,0 +1,28 @@ +//go:build libdns_rfc2136 || libdns_all +// +build libdns_rfc2136 libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/rfc2136" +) + +func init() { + module.Register("libdns.rfc2136", func(modName, instName string, _, _ []string) (module.Module, error) { + p := rfc2136.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("key_name", false, true, "", &p.KeyName) + c.String("key", false, true, "", &p.Key) + c.String("key_alg", false, true, "", &p.KeyAlg) + c.String("server", false, true, "", &p.Server) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/route53.go b/internal/libdns/route53.go new file mode 100644 index 0000000..dd50724 --- /dev/null +++ b/internal/libdns/route53.go @@ -0,0 +1,26 @@ +//go:build libdns_route53 || libdns_all +// +build libdns_route53 libdns_all + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/route53" +) + +func init() { + module.Register("libdns.route53", func(modName, instName string, _, _ []string) (module.Module, error) { + p := route53.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("secret_access_key", false, false, "", &p.SecretAccessKey) + c.String("access_key_id", false, false, "", &p.AccessKeyId) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/libdns/vultr.go b/internal/libdns/vultr.go new file mode 100644 index 0000000..9157258 --- /dev/null +++ b/internal/libdns/vultr.go @@ -0,0 +1,25 @@ +//go:build libdns_vultr || !libdns_separate +// +build libdns_vultr !libdns_separate + +package libdns + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/libdns/vultr" +) + +func init() { + module.Register("libdns.vultr", func(modName, instName string, _, _ []string) (module.Module, error) { + p := vultr.Provider{} + return &ProviderModule{ + RecordDeleter: &p, + RecordAppender: &p, + setConfig: func(c *config.Map) { + c.String("api_token", false, false, "", &p.APIToken) + }, + instName: instName, + modName: modName, + }, nil + }) +} diff --git a/internal/limits/limiters/bucket.go b/internal/limits/limiters/bucket.go new file mode 100644 index 0000000..ae653b1 --- /dev/null +++ b/internal/limits/limiters/bucket.go @@ -0,0 +1,153 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package limiters + +import ( + "context" + "sync" + "time" +) + +// BucketSet combines a group of Ls into a single key-indexed structure. +// Basically, each unique key gets its own counter. The main use case for +// BucketSet is to apply per-resource rate limiting. +// +// Amount of buckets is limited to a certain value. When the size of internal +// map is around or equal to that value, next Take call will attempt to remove +// any stale buckets from the group. If it is not possible to do so (all +// buckets are in active use), Take will return false. Alternatively, in some +// rare cases, some other (undefined) waiting Take can return false. +// +// A BucksetSet without a New function assigned is no-op: Take and TakeContext +// always succeed and Release does nothing. +type BucketSet struct { + // New function is used to construct underlying L instances. + // + // It is safe to change it only when BucketSet is not used by any + // goroutine. + New func() L + + // Time after which bucket is considered stale and can be removed from the + // set. For safe use with Rate limiter, it should be at least as twice as + // big as Rate refill interval. + ReapInterval time.Duration + + MaxBuckets int + + mLck sync.Mutex + m map[string]*struct { + r L + lastUse time.Time + } +} + +func NewBucketSet(new_ func() L, reapInterval time.Duration, maxBuckets int) *BucketSet { + return &BucketSet{ + New: new_, + ReapInterval: reapInterval, + MaxBuckets: maxBuckets, + m: map[string]*struct { + r L + lastUse time.Time + }{}, + } +} + +func (r *BucketSet) Close() { + r.mLck.Lock() + defer r.mLck.Unlock() + + for _, v := range r.m { + v.r.Close() + } +} + +func (r *BucketSet) take(key string) L { + r.mLck.Lock() + defer r.mLck.Unlock() + + if len(r.m) > r.MaxBuckets { + now := time.Now() + // Attempt to get rid of stale buckets. + for k, v := range r.m { + if v.lastUse.Sub(now) > r.ReapInterval { + // Drop the bucket, if there happen to be any waiting Take for it. + // It will return 'false', but this is fine for us since this + // whole 'reaping' process will run only when we are under a + // high load and dropping random requests in this case is a + // more or less reasonable thing to do. + v.r.Close() + delete(r.m, k) + } + } + + // Still full? E.g. all buckets are in use. + if len(r.m) > r.MaxBuckets { + return nil + } + } + + bucket, ok := r.m[key] + if !ok { + r.m[key] = &struct { + r L + lastUse time.Time + }{ + r: r.New(), + lastUse: time.Now(), + } + bucket = r.m[key] + } + r.m[key].lastUse = time.Now() + + return bucket.r +} + +func (r *BucketSet) Take(key string) bool { + if r.New == nil { + return true + } + + bucket := r.take(key) + return bucket.Take() +} + +func (r *BucketSet) Release(key string) { + if r.New == nil { + return + } + + r.mLck.Lock() + defer r.mLck.Unlock() + + bucket, ok := r.m[key] + if !ok { + return + } + bucket.r.Release() +} + +func (r *BucketSet) TakeContext(ctx context.Context, key string) error { + if r.New == nil { + return nil + } + + bucket := r.take(key) + return bucket.TakeContext(ctx) +} diff --git a/internal/limits/limiters/concurrency.go b/internal/limits/limiters/concurrency.go new file mode 100644 index 0000000..18923e8 --- /dev/null +++ b/internal/limits/limiters/concurrency.go @@ -0,0 +1,68 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package limiters + +import "context" + +// Semaphore is a convenience wrapper for a channel that implements +// semaphore-kind synchronization. +// +// If the argument given to the NewSemaphore is negative or zero, +// all methods are no-op. +type Semaphore struct { + c chan struct{} +} + +func NewSemaphore(max int) Semaphore { + return Semaphore{c: make(chan struct{}, max)} +} + +func (s Semaphore) Take() bool { + if cap(s.c) <= 0 { + return true + } + s.c <- struct{}{} + return true +} + +func (s Semaphore) TakeContext(ctx context.Context) error { + if cap(s.c) <= 0 { + return nil + } + select { + case s.c <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s Semaphore) Release() { + if cap(s.c) <= 0 { + return + } + select { + case <-s.c: + default: + panic("limiters: mismatched Release call") + } +} + +func (s Semaphore) Close() { +} diff --git a/internal/limits/limiters/limiters.go b/internal/limits/limiters/limiters.go new file mode 100644 index 0000000..9b11761 --- /dev/null +++ b/internal/limits/limiters/limiters.go @@ -0,0 +1,35 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package limiters provides a set of wrappers intended to restrict the amount +// of resources consumed by the server. +package limiters + +import "context" + +// The L interface represents a blocking limiter that has some upper bound of +// resource use and blocks when it is exceeded until enough resources are +// freed. +type L interface { + Take() bool + TakeContext(context.Context) error + Release() + + // Close frees any resources used internally by Limiter for book-keeping. + Close() +} diff --git a/internal/limits/limiters/multilimit.go b/internal/limits/limiters/multilimit.go new file mode 100644 index 0000000..d4f181c --- /dev/null +++ b/internal/limits/limiters/multilimit.go @@ -0,0 +1,69 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package limiters + +import "context" + +// MultiLimit wraps multiple L implementations into a single one, locking them +// in the specified order. +// +// It does not implement any deadlock detection or avoidance algorithms. +type MultiLimit struct { + Wrapped []L +} + +func (ml *MultiLimit) Take() bool { + for i := 0; i < len(ml.Wrapped); i++ { + if !ml.Wrapped[i].Take() { + // Acquire failed, undo acquire for all other resources we already + // got. + for _, l := range ml.Wrapped[:i] { + l.Release() + } + return false + } + } + return true +} + +func (ml *MultiLimit) TakeContext(ctx context.Context) error { + for i := 0; i < len(ml.Wrapped); i++ { + if err := ml.Wrapped[i].TakeContext(ctx); err != nil { + // Acquire failed, undo acquire for all other resources we already + // got. + for _, l := range ml.Wrapped[:i] { + l.Release() + } + return err + } + } + return nil +} + +func (ml *MultiLimit) Release() { + for _, l := range ml.Wrapped { + l.Release() + } +} + +func (ml *MultiLimit) Close() { + for _, l := range ml.Wrapped { + l.Close() + } +} diff --git a/internal/limits/limiters/rate.go b/internal/limits/limiters/rate.go new file mode 100644 index 0000000..a774187 --- /dev/null +++ b/internal/limits/limiters/rate.go @@ -0,0 +1,117 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package limiters + +import ( + "context" + "errors" + "time" +) + +var ErrClosed = errors.New("limiters: Rate bucket is closed") + +// Rate structure implements a basic rate-limiter for requests using the token +// bucket approach. +// +// Take() is expected to be called before each request. Excessive calls will +// block. Timeouts can be implemented using the TakeContext method. +// +// Rate.Close causes all waiting Take to return false. TakeContext returns +// ErrClosed in this case. +// +// If burstSize = 0, all methods are no-op and always succeed. +type Rate struct { + bucket chan struct{} + stop chan struct{} +} + +func NewRate(burstSize int, interval time.Duration) Rate { + r := Rate{ + bucket: make(chan struct{}, burstSize), + stop: make(chan struct{}), + } + + if burstSize == 0 { + return r + } + + for i := 0; i < burstSize; i++ { + r.bucket <- struct{}{} + } + + go r.fill(burstSize, interval) + return r +} + +func (r Rate) fill(burstSize int, interval time.Duration) { + t := time.NewTimer(interval) + defer t.Stop() + for { + t.Reset(interval) + select { + case <-t.C: + case <-r.stop: + close(r.bucket) + return + } + + fill: + for i := 0; i < burstSize; i++ { + select { + case r.bucket <- struct{}{}: + default: + // If there are no Take pending and the bucket is already + // full - don't block. + break fill + } + } + } +} + +func (r Rate) Take() bool { + if cap(r.bucket) == 0 { + return true + } + + _, ok := <-r.bucket + return ok +} + +func (r Rate) TakeContext(ctx context.Context) error { + if cap(r.bucket) == 0 { + return nil + } + + select { + case _, ok := <-r.bucket: + if !ok { + return ErrClosed + } + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (r Rate) Release() { +} + +func (r Rate) Close() { + close(r.stop) +} diff --git a/internal/limits/limits.go b/internal/limits/limits.go new file mode 100644 index 0000000..95d98a2 --- /dev/null +++ b/internal/limits/limits.go @@ -0,0 +1,234 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package limit provides a module object that can be used to restrict the +// concurrency and rate of the messages flow globally or on per-source, +// per-destination basis. +// +// Note, all domain inputs are interpreted with the assumption they are already +// normalized. +// +// Low-level components are available in the limiters/ subpackage. +package limits + +import ( + "context" + "net" + "strconv" + "time" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/limits/limiters" +) + +type Group struct { + instName string + + global limiters.MultiLimit + ip *limiters.BucketSet // BucketSet of MultiLimit + source *limiters.BucketSet // BucketSet of MultiLimit + dest *limiters.BucketSet // BucketSet of MultiLimit +} + +func New(_, instName string, _, _ []string) (module.Module, error) { + return &Group{ + instName: instName, + }, nil +} + +func (g *Group) Init(cfg *config.Map) error { + var ( + globalL []limiters.L + ipL []func() limiters.L + sourceL []func() limiters.L + destL []func() limiters.L + ) + + for _, child := range cfg.Block.Children { + if len(child.Args) < 1 { + return config.NodeErr(child, "at least two arguments are required") + } + + var ( + ctor func() limiters.L + err error + ) + switch kind := child.Args[0]; kind { + case "rate": + ctor, err = rateCtor(child, child.Args[1:]) + case "concurrency": + ctor, err = concurrencyCtor(child, child.Args[1:]) + default: + return config.NodeErr(child, "unknown limit kind: %v", kind) + } + if err != nil { + return err + } + + switch scope := child.Name; scope { + case "all": + globalL = append(globalL, ctor()) + case "ip": + ipL = append(ipL, ctor) + case "source": + sourceL = append(sourceL, ctor) + case "destination": + destL = append(destL, ctor) + default: + return config.NodeErr(child, "unknown limit scope: %v", scope) + } + } + + // 20010 is slightly higher than the default max. recipients count in + // endpoint/smtp. + g.global = limiters.MultiLimit{Wrapped: globalL} + if len(ipL) != 0 { + g.ip = limiters.NewBucketSet(func() limiters.L { + l := make([]limiters.L, 0, len(ipL)) + for _, ctor := range ipL { + l = append(l, ctor()) + } + return &limiters.MultiLimit{Wrapped: l} + }, 1*time.Minute, 20010) + } + if len(sourceL) != 0 { + g.source = limiters.NewBucketSet(func() limiters.L { + l := make([]limiters.L, 0, len(sourceL)) + for _, ctor := range sourceL { + l = append(l, ctor()) + } + return &limiters.MultiLimit{Wrapped: l} + }, 1*time.Minute, 20010) + } + if len(destL) != 0 { + g.dest = limiters.NewBucketSet(func() limiters.L { + l := make([]limiters.L, 0, len(sourceL)) + for _, ctor := range sourceL { + l = append(l, ctor()) + } + return &limiters.MultiLimit{Wrapped: l} + }, 1*time.Minute, 20010) + } + + return nil +} + +func rateCtor(node config.Node, args []string) (func() limiters.L, error) { + period := 1 * time.Second + burst := 0 + + switch len(args) { + case 2: + var err error + period, err = time.ParseDuration(args[1]) + if err != nil { + return nil, config.NodeErr(node, "%v", err) + } + fallthrough + case 1: + var err error + burst, err = strconv.Atoi(args[0]) + if err != nil { + return nil, config.NodeErr(node, "%v", err) + } + case 0: + return nil, config.NodeErr(node, "at least burst size is needed") + default: + return nil, config.NodeErr(node, "too many arguments") + } + + return func() limiters.L { + return limiters.NewRate(burst, period) + }, nil +} + +func concurrencyCtor(node config.Node, args []string) (func() limiters.L, error) { + if len(args) != 1 { + return nil, config.NodeErr(node, "max concurrency value is needed") + } + max, err := strconv.Atoi(args[0]) + if err != nil { + return nil, config.NodeErr(node, "%v", err) + } + return func() limiters.L { + return limiters.NewSemaphore(max) + }, nil +} + +func (g *Group) TakeMsg(ctx context.Context, addr net.IP, sourceDomain string) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + if err := g.global.TakeContext(ctx); err != nil { + return err + } + + if g.ip != nil { + if err := g.ip.TakeContext(ctx, addr.String()); err != nil { + g.global.Release() + return err + } + } + if g.source != nil { + if err := g.source.TakeContext(ctx, sourceDomain); err != nil { + g.global.Release() + g.ip.Release(addr.String()) + return err + } + } + return nil +} + +func (g *Group) TakeDest(ctx context.Context, domain string) error { + if g.dest == nil { + return nil + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + return g.dest.TakeContext(ctx, domain) +} + +func (g *Group) ReleaseMsg(addr net.IP, sourceDomain string) { + g.global.Release() + if g.ip != nil { + g.ip.Release(addr.String()) + } + if g.source != nil { + g.source.Release(sourceDomain) + } +} + +func (g *Group) ReleaseDest(domain string) { + if g.dest == nil { + return + } + g.dest.Release(domain) +} + +func (g *Group) Name() string { + return "limits" +} + +func (g *Group) InstanceName() string { + return g.instName +} + +func init() { + module.Register("limits", New) +} diff --git a/internal/modify/dkim/dkim.go b/internal/modify/dkim/dkim.go new file mode 100644 index 0000000..ffeed4a --- /dev/null +++ b/internal/modify/dkim/dkim.go @@ -0,0 +1,375 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dkim + +import ( + "context" + "crypto" + "errors" + "fmt" + "io" + "path/filepath" + "runtime/trace" + "strings" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/dkim" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" + "golang.org/x/net/idna" +) + +const Day = 86400 * time.Second + +var ( + oversignDefault = []string{ + // Directly visible to the user. + "Subject", + "Sender", + "To", + "Cc", + "From", + "Date", + + // Affects body processing. + "MIME-Version", + "Content-Type", + "Content-Transfer-Encoding", + + // Affects user interaction. + "Reply-To", + "In-Reply-To", + "Message-Id", + "References", + + // Provide additional security benefit for OpenPGP. + "Autocrypt", + "Openpgp", + } + signDefault = []string{ + // Mailing list information. Not oversigned to prevent signature + // breakage by aliasing MLMs. + "List-Id", + "List-Help", + "List-Unsubscribe", + "List-Post", + "List-Owner", + "List-Archive", + + // Not oversigned since it can be prepended by intermediate relays. + "Resent-To", + "Resent-Sender", + "Resent-Message-Id", + "Resent-Date", + "Resent-From", + "Resent-Cc", + } + + hashFuncs = map[string]crypto.Hash{ + "sha256": crypto.SHA256, + } +) + +type Modifier struct { + instName string + + domains []string + selector string + signers map[string]crypto.Signer + oversignHeader []string + signHeader []string + headerCanon dkim.Canonicalization + bodyCanon dkim.Canonicalization + sigExpiry time.Duration + hash crypto.Hash + multipleFromOk bool + signSubdomains bool + + log log.Logger +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + m := &Modifier{ + instName: instName, + signers: map[string]crypto.Signer{}, + log: log.Logger{Name: "modify.dkim"}, + } + + if len(inlineArgs) == 0 { + return m, nil + } + if len(inlineArgs) == 1 { + return nil, errors.New("modify.dkim: at least two arguments required") + } + + m.domains = inlineArgs[0 : len(inlineArgs)-1] + m.selector = inlineArgs[len(inlineArgs)-1] + + return m, nil +} + +func (m *Modifier) Name() string { + return "modify.dkim" +} + +func (m *Modifier) InstanceName() string { + return m.instName +} + +func (m *Modifier) Init(cfg *config.Map) error { + var ( + hashName string + keyPathTemplate string + newKeyAlgo string + ) + + cfg.Bool("debug", true, false, &m.log.Debug) + cfg.StringList("domains", false, false, m.domains, &m.domains) + cfg.String("selector", false, false, m.selector, &m.selector) + cfg.String("key_path", false, false, "dkim_keys/{domain}_{selector}.key", &keyPathTemplate) + cfg.StringList("oversign_fields", false, false, oversignDefault, &m.oversignHeader) + cfg.StringList("sign_fields", false, false, signDefault, &m.signHeader) + cfg.Enum("header_canon", false, false, + []string{string(dkim.CanonicalizationRelaxed), string(dkim.CanonicalizationSimple)}, + dkim.CanonicalizationRelaxed, (*string)(&m.headerCanon)) + cfg.Enum("body_canon", false, false, + []string{string(dkim.CanonicalizationRelaxed), string(dkim.CanonicalizationSimple)}, + dkim.CanonicalizationRelaxed, (*string)(&m.bodyCanon)) + cfg.Duration("sig_expiry", false, false, 5*Day, &m.sigExpiry) + cfg.Enum("hash", false, false, + []string{"sha256"}, "sha256", &hashName) + cfg.Enum("newkey_algo", false, false, + []string{"rsa4096", "rsa2048", "ed25519"}, "rsa2048", &newKeyAlgo) + cfg.Bool("allow_multiple_from", false, false, &m.multipleFromOk) + cfg.Bool("sign_subdomains", false, false, &m.signSubdomains) + + if _, err := cfg.Process(); err != nil { + return err + } + + if len(m.domains) == 0 { + return errors.New("sign_domain: at least one domain is needed") + } + if m.selector == "" { + return errors.New("sign_domain: selector is not specified") + } + if m.signSubdomains && len(m.domains) > 1 { + return errors.New("sign_domain: only one domain is supported when sign_subdomains is enabled") + } + + m.hash = hashFuncs[hashName] + if m.hash == 0 { + panic("modify.dkim.Init: Hash function allowed by config matcher but not present in hashFuncs") + } + + for _, domain := range m.domains { + if _, err := idna.ToASCII(domain); err != nil { + m.log.Printf("warning: unable to convert domain %s to A-labels form, non-EAI messages will not be signed: %v", domain, err) + } + + keyValues := strings.NewReplacer("{domain}", domain, "{selector}", m.selector) + keyPath := keyValues.Replace(keyPathTemplate) + + signer, newKey, err := m.loadOrGenerateKey(keyPath, newKeyAlgo) + if err != nil { + return err + } + + if newKey { + dnsPath := keyPath + ".dns" + if filepath.Ext(keyPath) == ".key" { + dnsPath = keyPath[:len(keyPath)-4] + ".dns" + } + m.log.Printf("generated a new %s keypair, private key is in %s, TXT record with public key is in %s,\n"+ + "put its contents into TXT record for %s._domainkey.%s to make signing and verification work", + newKeyAlgo, keyPath, dnsPath, m.selector, domain) + } + + normDomain, err := dns.ForLookup(domain) + if err != nil { + return fmt.Errorf("sign_skim: unable to normalize domain %s: %w", domain, err) + } + m.signers[normDomain] = signer + } + + return nil +} + +func (m *Modifier) fieldsToSign(h *textproto.Header) []string { + // Filter out duplicated fields from configs so they + // will not cause panic() in go-msgauth internals. + seen := make(map[string]struct{}) + + res := make([]string, 0, len(m.oversignHeader)+len(m.signHeader)) + for _, key := range m.oversignHeader { + if _, ok := seen[strings.ToLower(key)]; ok { + continue + } + seen[strings.ToLower(key)] = struct{}{} + + // Add to signing list once per each key use. + for field := h.FieldsByKey(key); field.Next(); { + res = append(res, key) + } + // And once more to "oversign" it. + res = append(res, key) + } + for _, key := range m.signHeader { + if _, ok := seen[strings.ToLower(key)]; ok { + continue + } + seen[strings.ToLower(key)] = struct{}{} + + // Add to signing list once per each key use. + for field := h.FieldsByKey(key); field.Next(); { + res = append(res, key) + } + } + return res +} + +type state struct { + m *Modifier + meta *module.MsgMetadata + from string + log log.Logger +} + +func (m *Modifier) ModStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.ModifierState, error) { + return &state{ + m: m, + meta: msgMeta, + log: target.DeliveryLogger(m.log, msgMeta), + }, nil +} + +func (s *state) RewriteSender(ctx context.Context, mailFrom string) (string, error) { + s.from = mailFrom + return mailFrom, nil +} + +func (s state) RewriteRcpt(ctx context.Context, rcptTo string) ([]string, error) { + return []string{rcptTo}, nil +} + +func (s *state) RewriteBody(ctx context.Context, h *textproto.Header, body buffer.Buffer) error { + defer trace.StartRegion(ctx, "modify.dkim/RewriteBody").End() + + var domain string + if s.from != "" { + var err error + _, domain, err = address.Split(s.from) + if err != nil { + return err + } + } + // Use first key for null return path (<>) and postmaster () + if domain == "" { + domain = s.m.domains[0] + } + selector := s.m.selector + + if s.m.signSubdomains { + topDomain := s.m.domains[0] + if strings.HasSuffix(domain, "."+topDomain) { + domain = topDomain + } + } + normDomain, err := dns.ForLookup(domain) + if err != nil { + s.log.Error("unable to normalize domain from envelope sender", err, "domain", domain) + return nil + } + keySigner := s.m.signers[normDomain] + if keySigner == nil { + s.log.Msg("no key for domain", "domain", normDomain) + return nil + } + + // If the message is non-EAI, we are not allowed to use domains in U-labels, + // attempt to convert. + if !s.meta.SMTPOpts.UTF8 { + var err error + domain, err = idna.ToASCII(domain) + if err != nil { + return nil + } + + selector, err = idna.ToASCII(selector) + if err != nil { + return nil + } + } + + opts := dkim.SignOptions{ + Domain: domain, + Selector: selector, + Identifier: "@" + domain, + Signer: keySigner, + Hash: s.m.hash, + HeaderCanonicalization: s.m.headerCanon, + BodyCanonicalization: s.m.bodyCanon, + HeaderKeys: s.m.fieldsToSign(h), + } + if s.m.sigExpiry != 0 { + opts.Expiration = time.Now().Add(s.m.sigExpiry) + } + signer, err := dkim.NewSigner(&opts) + if err != nil { + return exterrors.WithFields(err, map[string]interface{}{"modifier": "modify.dkim"}) + } + if err := textproto.WriteHeader(signer, *h); err != nil { + signer.Close() + return exterrors.WithFields(err, map[string]interface{}{"modifier": "modify.dkim"}) + } + r, err := body.Open() + if err != nil { + signer.Close() + return exterrors.WithFields(err, map[string]interface{}{"modifier": "modify.dkim"}) + } + if _, err := io.Copy(signer, r); err != nil { + signer.Close() + return exterrors.WithFields(err, map[string]interface{}{"modifier": "modify.dkim"}) + } + + if err := signer.Close(); err != nil { + return exterrors.WithFields(err, map[string]interface{}{"modifier": "modify.dkim"}) + } + + h.AddRaw([]byte(signer.Signature())) + + s.m.log.DebugMsg("signed", "domain", domain) + + return nil +} + +func (s state) Close() error { + return nil +} + +func init() { + module.Register("modify.dkim", New) +} diff --git a/internal/modify/dkim/dkim_test.go b/internal/modify/dkim/dkim_test.go new file mode 100644 index 0000000..d4a9b5a --- /dev/null +++ b/internal/modify/dkim/dkim_test.go @@ -0,0 +1,245 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dkim + +import ( + "bytes" + "context" + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/dkim" + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func newTestModifier(t *testing.T, dir, keyAlgo string, domains []string) *Modifier { + mod, err := New("", "test", nil, nil) + if err != nil { + t.Fatal(err) + } + m := mod.(*Modifier) + m.log = testutils.Logger(t, m.Name()) + + err = m.Init(config.NewMap(nil, config.Node{ + Children: []config.Node{ + { + Name: "domains", + Args: domains, + }, + { + Name: "selector", + Args: []string{"default"}, + }, + { + Name: "key_path", + Args: []string{filepath.Join(dir, "{domain}.key")}, + }, + { + Name: "newkey_algo", + Args: []string{keyAlgo}, + }, + }, + })) + if err != nil { + t.Fatal(err) + } + + return m +} + +func signTestMsg(t *testing.T, m *Modifier, envelopeFrom string) (textproto.Header, []byte) { + t.Helper() + + state, err := m.ModStateForMsg(context.Background(), &module.MsgMetadata{}) + if err != nil { + t.Fatal(err) + } + + testHdr := textproto.Header{} + testHdr.Add("From", "") + testHdr.Add("Subject", "heya") + testHdr.Add("To", "") + body := []byte("hello there\r\n") + + // modify.dkim expects RewriteSender to be called to get envelope sender + // (see module.Modifier docs) + + // RewriteSender does not fail for modify.dkim. It just sets envelopeFrom. + if _, err := state.RewriteSender(context.Background(), envelopeFrom); err != nil { + panic(err) + } + err = state.RewriteBody(context.Background(), &testHdr, buffer.MemoryBuffer{Slice: body}) + if err != nil { + t.Fatal(err) + } + + return testHdr, body +} + +func verifyTestMsg(t *testing.T, keysPath string, expectedDomains []string, hdr textproto.Header, body []byte) { + t.Helper() + + domainsMap := make(map[string]bool) + zones := map[string]mockdns.Zone{} + for _, domain := range expectedDomains { + dnsRecord, err := os.ReadFile(filepath.Join(keysPath, domain+".dns")) + if err != nil { + t.Fatal(err) + } + + t.Log("DNS record:", string(dnsRecord)) + zones["default._domainkey."+domain+"."] = mockdns.Zone{TXT: []string{string(dnsRecord)}} + domainsMap[domain] = false + } + + var fullBody bytes.Buffer + if err := textproto.WriteHeader(&fullBody, hdr); err != nil { + t.Fatal(err) + } + if _, err := fullBody.Write(body); err != nil { + t.Fatal(err) + } + + resolver := &mockdns.Resolver{Zones: zones} + verifs, err := dkim.VerifyWithOptions(bytes.NewReader(fullBody.Bytes()), &dkim.VerifyOptions{ + LookupTXT: func(domain string) ([]string, error) { + return resolver.LookupTXT(context.Background(), domain) + }, + }) + if err != nil { + t.Fatal(err) + } + for _, v := range verifs { + if v.Err != nil { + t.Errorf("Verification error for %s: %v", v.Domain, v.Err) + } + if _, ok := domainsMap[v.Domain]; !ok { + t.Errorf("Unexpected verification for domain %s", v.Domain) + } + + domainsMap[v.Domain] = true + } + for domain, ok := range domainsMap { + if !ok { + t.Errorf("Missing verification for domain %s", domain) + } + } +} + +func TestGenerateSignVerify(t *testing.T) { + // This test verifies whether a freshly generated key can be used for + // signing and verification. + // + // It is a kind of "integration" test for DKIM modifier, as it tests + // whether everything works correctly together. + // + // Additionally it also tests whether key selection works correctly. + + test := func(domains []string, envelopeFrom string, expectDomain []string, keyAlgo string, headerCanon, bodyCanon dkim.Canonicalization, reload bool) { + t.Helper() + + dir := t.TempDir() + + m := newTestModifier(t, dir, keyAlgo, domains) + m.bodyCanon = bodyCanon + m.headerCanon = headerCanon + if reload { + m = newTestModifier(t, dir, keyAlgo, domains) + } + + testHdr, body := signTestMsg(t, m, envelopeFrom) + verifyTestMsg(t, dir, expectDomain, testHdr, body) + } + + for _, algo := range [2]string{"rsa2048", "ed25519"} { + for _, hdrCanon := range [2]dkim.Canonicalization{dkim.CanonicalizationSimple, dkim.CanonicalizationRelaxed} { + for _, bodyCanon := range [2]dkim.Canonicalization{dkim.CanonicalizationSimple, dkim.CanonicalizationRelaxed} { + test([]string{"maddy.test"}, "test@maddy.test", []string{"maddy.test"}, algo, hdrCanon, bodyCanon, false) + test([]string{"maddy.test"}, "test@maddy.test", []string{"maddy.test"}, algo, hdrCanon, bodyCanon, true) + } + } + } + + // Key selection tests + test( + []string{"maddy.test"}, // Generated keys. + "test@maddy.test", // Envelope sender. + []string{"maddy.test"}, // Expected signature domains. + "ed25519", dkim.CanonicalizationRelaxed, dkim.CanonicalizationRelaxed, false) + test( + []string{"maddy.test"}, + "test@unrelated.maddy.test", + []string{}, + "ed25519", dkim.CanonicalizationRelaxed, dkim.CanonicalizationRelaxed, false) + test( + []string{"maddy.test", "related.maddy.test"}, + "test@related.maddy.test", + []string{"related.maddy.test"}, + "ed25519", dkim.CanonicalizationRelaxed, dkim.CanonicalizationRelaxed, false) + test( + []string{"fallback.maddy.test", "maddy.test"}, + "postmaster", + []string{"fallback.maddy.test"}, + "ed25519", dkim.CanonicalizationRelaxed, dkim.CanonicalizationRelaxed, false) + test( + []string{"fallback.maddy.test", "maddy.test"}, + "", + []string{"fallback.maddy.test"}, + "ed25519", dkim.CanonicalizationRelaxed, dkim.CanonicalizationRelaxed, false) + test( + []string{"another.maddy.test", "another.maddy.test", "maddy.test"}, + "test@another.maddy.test", + []string{"another.maddy.test"}, + "ed25519", dkim.CanonicalizationRelaxed, dkim.CanonicalizationRelaxed, false) + test( + []string{"another.maddy.test", "another.maddy.test", "maddy.test"}, + "", + []string{"another.maddy.test"}, + "ed25519", dkim.CanonicalizationRelaxed, dkim.CanonicalizationRelaxed, false) +} + +func TestFieldsToSign(t *testing.T) { + h := textproto.Header{} + h.Add("A", "1") + h.Add("c", "2") + h.Add("C", "3") + h.Add("a", "4") + h.Add("b", "5") + h.Add("unrelated", "6") + + m := Modifier{ + oversignHeader: []string{"A", "B"}, + signHeader: []string{"C"}, + } + fields := m.fieldsToSign(&h) + sort.Strings(fields) + expected := []string{"A", "A", "A", "B", "B", "C", "C"} + + if !reflect.DeepEqual(fields, expected) { + t.Errorf("incorrect set of fields to sign\nwant: %v\ngot: %v", expected, fields) + } +} diff --git a/internal/modify/dkim/keys.go b/internal/modify/dkim/keys.go new file mode 100644 index 0000000..7c39b76 --- /dev/null +++ b/internal/modify/dkim/keys.go @@ -0,0 +1,184 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dkim + +import ( + "crypto" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + "io" + "os" + "path/filepath" +) + +func (m *Modifier) loadOrGenerateKey(keyPath, newKeyAlgo string) (pkey crypto.Signer, newKey bool, err error) { + f, err := os.Open(keyPath) + if err != nil { + if os.IsNotExist(err) { + pkey, err = m.generateAndWrite(keyPath, newKeyAlgo) + return pkey, true, err + } + return nil, false, err + } + defer f.Close() + + pemBlob, err := io.ReadAll(f) + if err != nil { + return nil, false, err + } + + block, _ := pem.Decode(pemBlob) + if block == nil { + return nil, false, fmt.Errorf("modify.dkim: %s: invalid PEM block", keyPath) + } + + var key interface{} + switch block.Type { + case "PRIVATE KEY": // RFC 5208 aka PKCS #8 + key, err = x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, false, fmt.Errorf("modify.dkim: %s: %w", keyPath, err) + } + case "RSA PRIVATE KEY": // RFC 3447 aka PKCS #1 + key, err = x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, false, fmt.Errorf("modify.dkim: %s: %w", keyPath, err) + } + case "EC PRIVATE KEY": // RFC 5915 + key, err = x509.ParseECPrivateKey(block.Bytes) + if err != nil { + return nil, false, fmt.Errorf("modify.dkim: %s: %w", keyPath, err) + } + default: + return nil, false, fmt.Errorf("modify.dkim: %s: not a private key or unsupported format", keyPath) + } + + switch key := key.(type) { + case *rsa.PrivateKey: + if err := key.Validate(); err != nil { + return nil, false, err + } + key.Precompute() + return key, false, nil + case ed25519.PrivateKey: + return key, false, nil + case *ecdsa.PublicKey: + return nil, false, fmt.Errorf("modify.dkim: %s: ECDSA keys are not supported", keyPath) + default: + return nil, false, fmt.Errorf("modify.dkim: %s: unknown key type: %T", keyPath, key) + } +} + +func (m *Modifier) generateAndWrite(keyPath, newKeyAlgo string) (crypto.Signer, error) { + wrapErr := func(err error) error { + return fmt.Errorf("modify.dkim: generate %s: %w", keyPath, err) + } + + m.log.Printf("generating a new %s keypair...", newKeyAlgo) + + var ( + pkey crypto.Signer + dkimName = newKeyAlgo + err error + ) + switch newKeyAlgo { + case "rsa4096": + dkimName = "rsa" + pkey, err = rsa.GenerateKey(rand.Reader, 4096) + case "rsa2048": + dkimName = "rsa" + pkey, err = rsa.GenerateKey(rand.Reader, 2048) + case "ed25519": + _, pkey, err = ed25519.GenerateKey(rand.Reader) + default: + err = fmt.Errorf("unknown key algorithm: %s", newKeyAlgo) + } + if err != nil { + return nil, wrapErr(err) + } + + keyBlob, err := x509.MarshalPKCS8PrivateKey(pkey) + if err != nil { + return nil, wrapErr(err) + } + + // 0777 because we have public keys in here too and they don't + // need protection. Individual private key files have 0600 perms. + if err := os.MkdirAll(filepath.Dir(keyPath), 0o777); err != nil { + return nil, wrapErr(err) + } + + _, err = writeDNSRecord(keyPath, dkimName, pkey) + if err != nil { + return nil, wrapErr(err) + } + + f, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return nil, wrapErr(err) + } + + if err := pem.Encode(f, &pem.Block{ + Type: "PRIVATE KEY", + Bytes: keyBlob, + }); err != nil { + return nil, wrapErr(err) + } + + return pkey, nil +} + +func writeDNSRecord(keyPath, dkimAlgoName string, pkey crypto.Signer) (string, error) { + var ( + keyBlob []byte + pubkey = pkey.Public() + ) + switch pubkey := pubkey.(type) { + case *rsa.PublicKey: + var err error + keyBlob, err = x509.MarshalPKIXPublicKey(pubkey) + if err != nil { + return "", err + } + case ed25519.PublicKey: + keyBlob = pubkey + default: + panic("modify.dkim.writeDNSRecord: unknown key algorithm") + } + + dnsPath := keyPath + ".dns" + if filepath.Ext(keyPath) == ".key" { + dnsPath = keyPath[:len(keyPath)-4] + ".dns" + } + dnsF, err := os.Create(dnsPath) + if err != nil { + return "", err + } + keyRecord := fmt.Sprintf("v=DKIM1; k=%s; p=%s", dkimAlgoName, base64.StdEncoding.EncodeToString(keyBlob)) + if _, err := io.WriteString(dnsF, keyRecord); err != nil { + return "", err + } + return dnsPath, nil +} diff --git a/internal/modify/dkim/keys_test.go b/internal/modify/dkim/keys_test.go new file mode 100644 index 0000000..ebe13ba --- /dev/null +++ b/internal/modify/dkim/keys_test.go @@ -0,0 +1,156 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package dkim + +import ( + "crypto/ed25519" + "crypto/rsa" + "encoding/base64" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestKeyLoad_new(t *testing.T) { + m := Modifier{} + m.log = testutils.Logger(t, m.Name()) + + dir := t.TempDir() + + signer, newKey, err := m.loadOrGenerateKey(filepath.Join(dir, "testkey.key"), "ed25519") + if err != nil { + t.Fatal(err) + } + if !newKey { + t.Fatal("newKey=false") + } + + recordBlob, err := os.ReadFile(filepath.Join(dir, "testkey.dns")) + if err != nil { + t.Fatal(err) + } + var keyBlob []byte + for _, part := range strings.Split(string(recordBlob), ";") { + part = strings.TrimSpace(part) + if strings.HasPrefix(part, "k=") { + if part != "k=ed25519" { + t.Fatalf("Wrong type of generated key, want ed25519, got %s", part) + } + } + if strings.HasPrefix(part, "p=") { + keyBlob, err = base64.StdEncoding.DecodeString(part[2:]) + if err != nil { + t.Fatal(err) + } + } + } + + blob := signer.Public().(ed25519.PublicKey) + if string(blob) != string(keyBlob) { + t.Fatal("wrong public key placed into record file") + } +} + +const pkeyEd25519 = `-----BEGIN PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIJG9zs4vi2MYNkL9gUQwlmBLCzDODIJ5/1CwTAZFDm5U +-----END PRIVATE KEY-----` + +const pubkeyEd25519 = `5TPcCxzVByMyRsMFs5Dx23pnxKilI+1UrGg0t+O2oZU=` + +func TestKeyLoad_existing_pkcs8(t *testing.T) { + m := Modifier{} + m.log = testutils.Logger(t, m.Name()) + + dir := t.TempDir() + + if err := os.WriteFile(filepath.Join(dir, "testkey.key"), []byte(pkeyEd25519), 0o600); err != nil { + t.Fatal(err) + } + + signer, newKey, err := m.loadOrGenerateKey(filepath.Join(dir, "testkey.key"), "ed25519") + if err != nil { + t.Fatal(err) + } + if newKey { + t.Fatal("newKey = true") + } + + blob := signer.Public().(ed25519.PublicKey) + if signerKey := base64.StdEncoding.EncodeToString(blob); signerKey != pubkeyEd25519 { + t.Fatalf("wrong public key returned by loadOrGenerateKey, \nwant %s\ngot %s", pubkeyEd25519, signerKey) + } +} + +const pkeyRSA = `-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAuxWwDR9ADiuV2b9xF+btOIgwS5W0yJeS/Dht4HlUELrye2JZ +7TCQpx2Hs1FY5Tkj4VLnYHTPftS6cLYNx6hQbWZMhj5qmP9ccQ8rqdgdLB5RqCn3 +zo8wbKFZ8ygYt1yZyNOfJLNTBjIcC1BCKoZosA7MWHUOwRtt1ARVmldsNH3iio0l +wHjyKNYd0Kqw4uGEg6sulK69lw4G8YTnKtCt0G8vCpQHyQepolOMF7Q1NZEw02/U +E54qgaaC+ym+BQsqqF5iodmuIfLX+W0kKDee2YYhjuxNaFcPhE5j35LlGHCsrL0X +h4+2VZSYXuAO5aWpwX9jrrSFyCJLD/aYGMgdrwIDAQABAoIBAEZrF2UZCidLSJA5 +evwgM9I/kM4if3Wxd+Xv54vCn13cwECo+GhLC2ebueRJDkjZhSPe7LBlx2RZ9gNO +w0kPlZZYFx3AiKcmF0mHCExZyEE++EVv5pKdWwDIiu73fLYn6MqqvRA3X1zJp7yq +bP1MskLyjwAMr40IIgLXztDVbykiRC2Rw+o5cu7o3e0p0sFqJsjCUKtXZuzLePOk +gYYZ4FsmmVYh7pf244NEQao+fT19RtFL85E17yAHv+YD7qUBdbxoWIuAher9N/C0 +vOj4xYbNxbkS0+BTbygLAog5mFtNbAGysUZZ3YOYfKYgj9/u+aKwr2ZS2zIEeJj0 +eAiHtWECgYEA48dqxrR76JyukHid+XyI4Nqt+2EHEeDi23WTTT6lSZL1F3I2q7FF +DSHOA3hGw57GAMNQYCSzYxC4TBpZwJ7/8NdhA/kJg7tLOqcvZtS3Bu5bzLqLOCqL +E1tgh2LrpWjit2v+VSsQlf+QjG7QAEiWtya+AOfNWenILfxk2VNPP3MCgYEA0kOM +ym/EcgcSSihbFyyYO4UHZZ7rWiPRB+BtatJbEADMXMlwSAXvvVCpWSZBKBKjIE2y +ZM+kvv50QUd4ue7dKVEnqOy26XuAmuTE14smx1QyNonRvBV/HItJ0tKfMIZbXOpq +S2ESXkFybCzdOfzWOhx0PHjr40w8XUeSZi0LodUCgYAsC8bhD8uaKpozA7AAq41I +deEI6DVWxrb3mx/V4xRRSuKsGwDpaIkixfOxhhOhBlXhleM4BEDQGk6ZIMtUTSrO +5scy3nhxick9WVD4QI/3/iWwTC5ZuRhVsOjUpVNOFB8rOu3eiEpXxyirj04Xj/Hd +DtfVEv4JsgRsqA7UW6DKcwKBgQCiCvMXFDnWEwMSabWBz5lmzWfc9jO1HUM8Ccbp +e0I4vBTDMW854nFXejF5BhVS18Il5BsmvCvgEePwZy9wQ9jnvaaN9hglKkv7k3Ds +GE6DcazdASvFAuAaVHJJao7Ka9E/c10FyMLKJzASlCTOSr+iu0kNTbelTZx72uvF +mNONHQKBgCEUuJMM11mV0FCsVfJsmIv6z/zqOiPiOVbP1Bv2WlVzipvkI9bm6OyN +VHO8+oqFWyhJ3qRzebuPIefL8U6xjfMshX8MB23cB0J5LTPDZH3LCSmFvjr942EK +5+ewYHKtmS+6aaE+J+oB11r7XU8FyEI0kv6rAPDwJ19K4BMG/x7J +-----END RSA PRIVATE KEY-----` + +func TestKeyLoad_existing_pkcs1(t *testing.T) { + m := Modifier{} + m.log = testutils.Logger(t, m.Name()) + + dir := t.TempDir() + + if err := os.WriteFile(filepath.Join(dir, "testkey.key"), []byte(pkeyRSA), 0o600); err != nil { + t.Fatal(err) + } + + signer, newKey, err := m.loadOrGenerateKey(filepath.Join(dir, "testkey.key"), "rsa2048") + if err != nil { + t.Fatal(err) + } + if newKey { + t.Fatal("newKey=true") + } + + pubkey := signer.Public().(*rsa.PublicKey) + if pubkey.E != 65537 { + t.Fatalf("wrong public key returned by loadOrGenerateKey, got %d", pubkey.E) + } + if pubkey.N.String() != "23617257632228188386824425094266725423560758883229529475904285522114491665694237598874002862630696077162868821164059728985148713872807170386818903503533709975391952347175641552635505497204925274569104682448177717429244936284920784061388978739927939000424446717818401440783667723710780854637197555911253613285419663410256437304926940168312631109994734698918250930969511949067760562140706765511288141008942649676427142664185811322596443990204153105455693515405445788622172538582060141770589195075185467867938584021491237815987395835392935511032761463924045865609068314478096903374718657496007822964380498648030935260591" { + t.Fatalf("wrong public key returned by loadOrGenerateKey, got %s", pubkey.N.String()) + } +} diff --git a/internal/modify/group.go b/internal/modify/group.go new file mode 100644 index 0000000..3278b0f --- /dev/null +++ b/internal/modify/group.go @@ -0,0 +1,140 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package modify + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/module" +) + +type ( + // Group wraps multiple modifiers and runs them serially. + // + // It is also registered as a module under 'modifiers' name and acts as a + // module group. + Group struct { + instName string + Modifiers []module.Modifier + } + + groupState struct { + states []module.ModifierState + } +) + +func (g *Group) Init(cfg *config.Map) error { + for _, node := range cfg.Block.Children { + mod, err := modconfig.MsgModifier(cfg.Globals, append([]string{node.Name}, node.Args...), node) + if err != nil { + return err + } + + g.Modifiers = append(g.Modifiers, mod) + } + + return nil +} + +func (g *Group) Name() string { + return "modifiers" +} + +func (g *Group) InstanceName() string { + return g.instName +} + +func (g Group) ModStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.ModifierState, error) { + gs := groupState{} + for _, modifier := range g.Modifiers { + state, err := modifier.ModStateForMsg(ctx, msgMeta) + if err != nil { + // Free state objects we initialized already. + for _, state := range gs.states { + state.Close() + } + return nil, err + } + gs.states = append(gs.states, state) + } + return gs, nil +} + +func (gs groupState) RewriteSender(ctx context.Context, mailFrom string) (string, error) { + var err error + for _, state := range gs.states { + mailFrom, err = state.RewriteSender(ctx, mailFrom) + if err != nil { + return "", err + } + } + return mailFrom, nil +} + +func (gs groupState) RewriteRcpt(ctx context.Context, rcptTo string) ([]string, error) { + var err error + var result = []string{rcptTo} + for _, state := range gs.states { + var intermediateResult = []string{} + for _, partResult := range result { + var partResult_multi []string + partResult_multi, err = state.RewriteRcpt(ctx, partResult) + if err != nil { + return []string{""}, err + } + intermediateResult = append(intermediateResult, partResult_multi...) + } + result = intermediateResult + } + return result, nil +} + +func (gs groupState) RewriteBody(ctx context.Context, h *textproto.Header, body buffer.Buffer) error { + for _, state := range gs.states { + if err := state.RewriteBody(ctx, h, body); err != nil { + return err + } + } + return nil +} + +func (gs groupState) Close() error { + // We still try close all state objects to minimize + // resource leaks when Close fails for one object.. + + var lastErr error + for _, state := range gs.states { + if err := state.Close(); err != nil { + lastErr = err + } + } + return lastErr +} + +func init() { + module.Register("modifiers", func(_, instName string, _, _ []string) (module.Module, error) { + return &Group{ + instName: instName, + }, nil + }) +} diff --git a/internal/modify/replace_addr.go b/internal/modify/replace_addr.go new file mode 100644 index 0000000..50dd5df --- /dev/null +++ b/internal/modify/replace_addr.go @@ -0,0 +1,156 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package modify + +import ( + "context" + "fmt" + "strings" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/module" +) + +// replaceAddr is a simple module that replaces matching sender (or recipient) address +// in messages using module.Table implementation. +// +// If created with modName = "modify.replace_sender", it will change sender address. +// If created with modName = "modify.replace_rcpt", it will change recipient addresses. +type replaceAddr struct { + modName string + instName string + inlineArgs []string + + replaceSender bool + replaceRcpt bool + table module.MultiTable +} + +func NewReplaceAddr(modName, instName string, _, inlineArgs []string) (module.Module, error) { + r := replaceAddr{ + modName: modName, + instName: instName, + inlineArgs: inlineArgs, + replaceSender: modName == "modify.replace_sender", + replaceRcpt: modName == "modify.replace_rcpt", + } + + return &r, nil +} + +func (r *replaceAddr) Init(cfg *config.Map) error { + return modconfig.ModuleFromNode("table", r.inlineArgs, cfg.Block, cfg.Globals, &r.table) +} + +func (r replaceAddr) Name() string { + return r.modName +} + +func (r replaceAddr) InstanceName() string { + return r.instName +} + +func (r replaceAddr) ModStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.ModifierState, error) { + return r, nil +} + +func (r replaceAddr) RewriteSender(ctx context.Context, mailFrom string) (string, error) { + if r.replaceSender { + results, err := r.rewrite(ctx, mailFrom) + if err != nil { + return mailFrom, err + } + mailFrom = results[0] + } + return mailFrom, nil +} + +func (r replaceAddr) RewriteRcpt(ctx context.Context, rcptTo string) ([]string, error) { + if r.replaceRcpt { + return r.rewrite(ctx, rcptTo) + } + return []string{rcptTo}, nil +} + +func (r replaceAddr) RewriteBody(ctx context.Context, h *textproto.Header, body buffer.Buffer) error { + return nil +} + +func (r replaceAddr) Close() error { + return nil +} + +func (r replaceAddr) rewrite(ctx context.Context, val string) ([]string, error) { + normAddr, err := address.ForLookup(val) + if err != nil { + return []string{val}, fmt.Errorf("malformed address: %v", err) + } + + replacements, err := r.table.LookupMulti(ctx, normAddr) + if err != nil { + return []string{val}, err + } + if len(replacements) > 0 { + for _, replacement := range replacements { + if !address.Valid(replacement) { + return []string{""}, fmt.Errorf("refusing to replace recipient with the invalid address %s", replacement) + } + } + return replacements, nil + } + + mbox, domain, err := address.Split(normAddr) + if err != nil { + // If we have malformed address here, something is really wrong, but let's + // ignore it silently then anyway. + return []string{val}, nil + } + + // mbox is already normalized, since it is a part of address.ForLookup + // result. + replacements, err = r.table.LookupMulti(ctx, mbox) + if err != nil { + return []string{val}, err + } + if len(replacements) > 0 { + var results = make([]string, len(replacements)) + for i, replacement := range replacements { + if strings.Contains(replacement, "@") && !strings.HasPrefix(replacement, `"`) && !strings.HasSuffix(replacement, `"`) { + if !address.Valid(replacement) { + return []string{""}, fmt.Errorf("refusing to replace recipient with invalid address %s", replacement) + } + results[i] = replacement + } else { + results[i] = replacement + "@" + domain + } + } + return results, nil + } + + return []string{val}, nil +} + +func init() { + module.Register("modify.replace_sender", NewReplaceAddr) + module.Register("modify.replace_rcpt", NewReplaceAddr) +} diff --git a/internal/modify/replace_addr_test.go b/internal/modify/replace_addr_test.go new file mode 100644 index 0000000..e16cfd3 --- /dev/null +++ b/internal/modify/replace_addr_test.go @@ -0,0 +1,114 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package modify + +import ( + "context" + "reflect" + "testing" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/internal/testutils" +) + +func testReplaceAddr(t *testing.T, modName string) { + test := func(addr string, expectedMulti []string, aliases map[string][]string) { + t.Helper() + + mod, err := NewReplaceAddr(modName, "", nil, []string{"dummy"}) + if err != nil { + t.Fatal(err) + } + m := mod.(*replaceAddr) + if err := m.Init(config.NewMap(nil, config.Node{})); err != nil { + t.Fatal(err) + } + m.table = testutils.MultiTable{M: aliases} + + var actualMulti []string + if modName == "modify.replace_sender" { + var actual string + actual, err = m.RewriteSender(context.Background(), addr) + if err != nil { + t.Fatal(err) + } + actualMulti = []string{actual} + } + if modName == "modify.replace_rcpt" { + actualMulti, err = m.RewriteRcpt(context.Background(), addr) + if err != nil { + t.Fatal(err) + } + } + + if !reflect.DeepEqual(actualMulti, expectedMulti) { + t.Errorf("want %s, got %s", expectedMulti, actualMulti) + } + } + + test("test@example.org", []string{"test@example.org"}, nil) + test("postmaster", []string{"postmaster"}, nil) + test("test@example.com", []string{"test@example.org"}, + map[string][]string{"test@example.com": []string{"test@example.org"}}) + test(`"\"test @ test\""@example.com`, []string{"test@example.org"}, + map[string][]string{`"\"test @ test\""@example.com`: []string{"test@example.org"}}) + test(`test@example.com`, []string{`"\"test @ test\""@example.org`}, + map[string][]string{`test@example.com`: []string{`"\"test @ test\""@example.org`}}) + test(`"\"test @ test\""@example.com`, []string{`"\"b @ b\""@example.com`}, + map[string][]string{`"\"test @ test\""`: []string{`"\"b @ b\""`}}) + test("TeSt@eXAMple.com", []string{"test@example.org"}, + map[string][]string{"test@example.com": []string{"test@example.org"}}) + test("test@example.com", []string{"test2@example.com"}, + map[string][]string{"test": []string{"test2"}}) + test("test@example.com", []string{"test2@example.org"}, + map[string][]string{"test": []string{"test2@example.org"}}) + test("postmaster", []string{"test2@example.org"}, + map[string][]string{"postmaster": []string{"test2@example.org"}}) + test("TeSt@examPLE.com", []string{"test2@example.com"}, + map[string][]string{"test": []string{"test2"}}) + test("test@example.com", []string{"test3@example.com"}, + map[string][]string{ + "test@example.com": []string{"test3@example.com"}, + "test": []string{"test2"}, + }) + test("rcpt@E\u0301.example.com", []string{"rcpt@foo.example.com"}, + map[string][]string{ + "rcpt@\u00E9.example.com": []string{"rcpt@foo.example.com"}, + }) + test("E\u0301@foo.example.com", []string{"rcpt@foo.example.com"}, + map[string][]string{ + "\u00E9@foo.example.com": []string{"rcpt@foo.example.com"}, + }) + + if modName == "modify.replace_rcpt" { + //multiple aliases + test("test@example.com", []string{"test@example.org", "test@example.net"}, + map[string][]string{"test@example.com": []string{"test@example.org", "test@example.net"}}) + test("test@example.com", []string{"1@example.com", "2@example.com", "3@example.com"}, + map[string][]string{"test@example.com": []string{"1@example.com", "2@example.com", "3@example.com"}}) + } +} + +func TestReplaceAddr_RewriteSender(t *testing.T) { + testReplaceAddr(t, "modify.replace_sender") +} + +func TestReplaceAddr_RewriteRcpt(t *testing.T) { + testReplaceAddr(t, "modify.replace_rcpt") +} diff --git a/internal/msgpipeline/bench_test.go b/internal/msgpipeline/bench_test.go new file mode 100644 index 0000000..ae9aa8e --- /dev/null +++ b/internal/msgpipeline/bench_test.go @@ -0,0 +1,98 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "strconv" + "testing" + + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func BenchmarkMsgPipelineSimple(b *testing.B) { + target := testutils.Target{InstName: "test_target", DiscardMessages: true} + d := MsgPipeline{msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }} + + testutils.BenchDelivery(b, &d, "sender@example.org", []string{"rcpt-X@example.org"}) +} + +func BenchmarkMsgPipelineGlobalChecks(b *testing.B) { + testWithCount := func(checksCount int) { + b.Run(strconv.Itoa(checksCount), func(b *testing.B) { + checks := make([]module.Check, 0, checksCount) + for i := 0; i < checksCount; i++ { + checks = append(checks, &testutils.Check{InstName: "check_" + strconv.Itoa(i)}) + } + + target := testutils.Target{InstName: "test_target", DiscardMessages: true} + d := MsgPipeline{msgpipelineCfg: msgpipelineCfg{ + globalChecks: checks, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }} + + testutils.BenchDelivery(b, &d, "sender@example.org", []string{"rcpt-X@example.org"}) + }) + } + + testWithCount(5) + testWithCount(10) + testWithCount(15) +} + +func BenchmarkMsgPipelineTargets(b *testing.B) { + testWithCount := func(targetCount int) { + b.Run(strconv.Itoa(targetCount), func(b *testing.B) { + targets := make([]module.DeliveryTarget, 0, targetCount) + for i := 0; i < targetCount; i++ { + targets = append(targets, &testutils.Target{InstName: "target_" + strconv.Itoa(i), DiscardMessages: true}) + } + + d := MsgPipeline{msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: targets, + }, + }, + }} + + testutils.BenchDelivery(b, &d, "sender@example.org", []string{"rcpt-X@example.org"}) + }) + } + + testWithCount(5) + testWithCount(10) + testWithCount(15) +} diff --git a/internal/msgpipeline/bodynonatomic_test.go b/internal/msgpipeline/bodynonatomic_test.go new file mode 100644 index 0000000..c441980 --- /dev/null +++ b/internal/msgpipeline/bodynonatomic_test.go @@ -0,0 +1,148 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "errors" + "testing" + + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/modify" + "github.com/foxcpp/maddy/internal/testutils" +) + +type multipleErrs map[string]error + +func (m multipleErrs) SetStatus(rcptTo string, err error) { + m[rcptTo] = err +} + +func TestMsgPipeline_BodyNonAtomic(t *testing.T) { + err := errors.New("go away") + + target := testutils.Target{ + PartialBodyErr: map[string]error{ + "tester@example.org": err, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + c := multipleErrs{} + testutils.DoTestDeliveryNonAtomic(t, c, &d, "sender@example.org", []string{"tester@example.org", "tester2@example.org"}) + + if c["tester@example.org"] == nil { + t.Fatalf("no error for tester@example.org") + } + if c["tester@example.org"].Error() != err.Error() { + t.Errorf("wrong error for tester@example.org: %v", err) + } +} + +func TestMsgPipeline_BodyNonAtomic_ModifiedRcpt(t *testing.T) { + err := errors.New("go away") + + target := testutils.Target{ + PartialBodyErr: map[string]error{ + "tester-alias@example.org": err, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{ + testutils.Modifier{ + InstName: "test_modifier", + RcptTo: map[string][]string{ + "tester@example.org": []string{"tester-alias@example.org"}, + }, + }, + }, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + c := multipleErrs{} + testutils.DoTestDeliveryNonAtomic(t, c, &d, "sender@example.org", []string{"tester@example.org"}) + + if c["tester@example.org"] == nil { + t.Fatalf("no error for tester@example.org") + } + if c["tester@example.org"].Error() != err.Error() { + t.Errorf("wrong error for tester@example.org: %v", err) + } +} + +func TestMsgPipeline_BodyNonAtomic_ExpandAtomic(t *testing.T) { + err := errors.New("go away") + + target, target2 := testutils.Target{ + PartialBodyErr: map[string]error{ + "tester@example.org": err, + }, + }, testutils.Target{ + BodyErr: err, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target, &target2}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + c := multipleErrs{} + testutils.DoTestDeliveryNonAtomic(t, c, &d, "sender@example.org", []string{"tester@example.org", "tester2@example.org"}) + + if c["tester@example.org"] == nil { + t.Fatalf("no error for tester@example.org") + } + if c["tester@example.org"].Error() != err.Error() { + t.Errorf("wrong error for tester@example.org: %v", err) + } + if c["tester2@example.org"] == nil { + t.Fatalf("no error for tester@example.org") + } + if c["tester2@example.org"].Error() != err.Error() { + t.Errorf("wrong error for tester@example.org: %v", err) + } +} diff --git a/internal/msgpipeline/check_group.go b/internal/msgpipeline/check_group.go new file mode 100644 index 0000000..1fd6b24 --- /dev/null +++ b/internal/msgpipeline/check_group.go @@ -0,0 +1,67 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/module" +) + +// CheckGroup is a module container for a group of Check implementations. +// +// It allows to share a set of filter configurations between using named +// configuration blocks (module instances) system. +// +// It is registered globally under the name 'checks'. The object does not +// implement any standard module interfaces besides module.Module and is +// specific to the message pipeline. +type CheckGroup struct { + instName string + L []module.Check +} + +func (cg *CheckGroup) Init(cfg *config.Map) error { + for _, node := range cfg.Block.Children { + chk, err := modconfig.MessageCheck(cfg.Globals, append([]string{node.Name}, node.Args...), node) + if err != nil { + return err + } + + cg.L = append(cg.L, chk) + } + + return nil +} + +func (CheckGroup) Name() string { + return "checks" +} + +func (cg CheckGroup) InstanceName() string { + return cg.instName +} + +func init() { + module.Register("checks", func(_, instName string, _, _ []string) (module.Module, error) { + return &CheckGroup{ + instName: instName, + }, nil + }) +} diff --git a/internal/msgpipeline/check_runner.go b/internal/msgpipeline/check_runner.go new file mode 100644 index 0000000..f6df952 --- /dev/null +++ b/internal/msgpipeline/check_runner.go @@ -0,0 +1,349 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "context" + "runtime/debug" + "sync" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/dmarc" +) + +// checkRunner runs groups of checks, collects and merges results. +// It also makes sure that each check gets only one state object created. +type checkRunner struct { + msgMeta *module.MsgMetadata + mailFrom string + mailFromReceived bool + + checkedRcpts []string + checkedRcptsPerCheck map[module.CheckState]map[string]struct{} + checkedRcptsLock sync.Mutex + + resolver dns.Resolver + doDMARC bool + didDMARCFetch bool + dmarcVerify *dmarc.Verifier + + log log.Logger + + states map[module.Check]module.CheckState + + mergedRes module.CheckResult +} + +func newCheckRunner(msgMeta *module.MsgMetadata, log log.Logger, r dns.Resolver) *checkRunner { + return &checkRunner{ + msgMeta: msgMeta, + checkedRcptsPerCheck: map[module.CheckState]map[string]struct{}{}, + log: log, + resolver: r, + dmarcVerify: dmarc.NewVerifier(r), + states: make(map[module.Check]module.CheckState), + } +} + +func (cr *checkRunner) checkStates(ctx context.Context, checks []module.Check) ([]module.CheckState, error) { + states := make([]module.CheckState, 0, len(checks)) + newStates := make([]module.CheckState, 0, len(checks)) + newStatesMap := make(map[module.Check]module.CheckState, len(checks)) + closeStates := func() { + for _, state := range states { + state.Close() + } + } + + for _, check := range checks { + state, ok := cr.states[check] + if ok { + states = append(states, state) + continue + } + + cr.log.Debugf("initializing state for %v (%p)", objectName(check), check) + state, err := check.CheckStateForMsg(ctx, cr.msgMeta) + if err != nil { + closeStates() + return nil, err + } + states = append(states, state) + newStates = append(newStates, state) + newStatesMap[check] = state + } + + if len(newStates) == 0 { + return states, nil + } + + // Here we replay previous CheckConnection/CheckSender/CheckRcpt calls + // for any newly initialized checks so they all get change to see all these things. + // + // Done outside of check loop above to make sure we can run these for multiple + // checks in parallel. + if cr.mailFromReceived { + err := cr.runAndMergeResults(newStates, func(s module.CheckState) module.CheckResult { + res := s.CheckConnection(ctx) + return res + }) + if err != nil { + closeStates() + return nil, err + } + err = cr.runAndMergeResults(newStates, func(s module.CheckState) module.CheckResult { + res := s.CheckSender(ctx, cr.mailFrom) + return res + }) + if err != nil { + closeStates() + return nil, err + } + } + + if len(cr.checkedRcpts) != 0 { + for _, rcpt := range cr.checkedRcpts { + err := cr.runAndMergeResults(states, func(s module.CheckState) module.CheckResult { + // Avoid calling CheckRcpt for the same recipient for the same check + // multiple times, even if requested. + cr.checkedRcptsLock.Lock() + if _, ok := cr.checkedRcptsPerCheck[s][rcpt]; ok { + cr.checkedRcptsLock.Unlock() + return module.CheckResult{} + } + if cr.checkedRcptsPerCheck[s] == nil { + cr.checkedRcptsPerCheck[s] = make(map[string]struct{}) + } + cr.checkedRcptsPerCheck[s][rcpt] = struct{}{} + cr.checkedRcptsLock.Unlock() + + res := s.CheckRcpt(ctx, rcpt) + return res + }) + if err != nil { + closeStates() + return nil, err + } + } + } + + // This is done after all actions that can fail so we will not have to remove + // state objects from main map. + for check, state := range newStatesMap { + cr.states[check] = state + } + + return states, nil +} + +func (cr *checkRunner) runAndMergeResults(states []module.CheckState, runner func(module.CheckState) module.CheckResult) error { + data := struct { + authResLock sync.Mutex + headerLock sync.Mutex + + quarantineErr error + quarantineCheck string + setQuarantineErr sync.Once + + rejectErr error + rejectCheck string + setRejectErr sync.Once + + wg sync.WaitGroup + }{} + + for _, state := range states { + data.wg.Add(1) + go func() { + defer func() { + data.wg.Done() + if err := recover(); err != nil { + stack := debug.Stack() + log.Printf("panic during check execution: %v\n%s", err, stack) + } + }() + + subCheckRes := runner(state) + + // We check the length because we don't want to take locks + // when it is not necessary. + if len(subCheckRes.AuthResult) != 0 { + data.authResLock.Lock() + cr.mergedRes.AuthResult = append(cr.mergedRes.AuthResult, subCheckRes.AuthResult...) + data.authResLock.Unlock() + } + if subCheckRes.Header.Len() != 0 { + data.headerLock.Lock() + for field := subCheckRes.Header.Fields(); field.Next(); { + formatted, err := field.Raw() + if err != nil { + cr.log.Error("malformed header field added by check", err) + } + cr.mergedRes.Header.AddRaw(formatted) + } + data.headerLock.Unlock() + } + + if subCheckRes.Quarantine { + data.setQuarantineErr.Do(func() { + data.quarantineErr = subCheckRes.Reason + }) + } else if subCheckRes.Reject { + data.setRejectErr.Do(func() { + data.rejectErr = subCheckRes.Reason + }) + } else if subCheckRes.Reason != nil { + // 'action ignore' case. There is Reason, but action.Apply set + // both Reject and Quarantine to false. Log the reason for + // purposes of deployment testing. + cr.log.Error("no check action", subCheckRes.Reason) + } + }() + } + + data.wg.Wait() + if data.rejectErr != nil { + return data.rejectErr + } + + if data.quarantineErr != nil { + cr.log.Error("quarantined", data.quarantineErr) + cr.mergedRes.Quarantine = true + } + + return nil +} + +func (cr *checkRunner) checkConnSender(ctx context.Context, checks []module.Check, mailFrom string) error { + cr.mailFrom = mailFrom + cr.mailFromReceived = true + + // checkStates will run CheckConnection and CheckSender. + _, err := cr.checkStates(ctx, checks) + return err +} + +func (cr *checkRunner) checkRcpt(ctx context.Context, checks []module.Check, rcptTo string) error { + states, err := cr.checkStates(ctx, checks) + if err != nil { + return err + } + + err = cr.runAndMergeResults(states, func(s module.CheckState) module.CheckResult { + cr.checkedRcptsLock.Lock() + if _, ok := cr.checkedRcptsPerCheck[s][rcptTo]; ok { + cr.checkedRcptsLock.Unlock() + return module.CheckResult{} + } + if cr.checkedRcptsPerCheck[s] == nil { + cr.checkedRcptsPerCheck[s] = make(map[string]struct{}) + } + cr.checkedRcptsPerCheck[s][rcptTo] = struct{}{} + cr.checkedRcptsLock.Unlock() + + res := s.CheckRcpt(ctx, rcptTo) + return res + }) + + cr.checkedRcpts = append(cr.checkedRcpts, rcptTo) + return err +} + +func (cr *checkRunner) checkBody(ctx context.Context, checks []module.Check, header textproto.Header, body buffer.Buffer) error { + states, err := cr.checkStates(ctx, checks) + if err != nil { + return err + } + + if cr.doDMARC && !cr.didDMARCFetch { + cr.dmarcVerify.FetchRecord(ctx, header) + cr.didDMARCFetch = true + } + + return cr.runAndMergeResults(states, func(s module.CheckState) module.CheckResult { + res := s.CheckBody(ctx, header, body) + return res + }) +} + +func (cr *checkRunner) applyResults(hostname string, header *textproto.Header) error { + if cr.mergedRes.Quarantine { + cr.msgMeta.Quarantine = true + } + + if cr.doDMARC { + dmarcRes, policy := cr.dmarcVerify.Apply(cr.mergedRes.AuthResult) + cr.mergedRes.AuthResult = append(cr.mergedRes.AuthResult, &dmarcRes.Authres) + switch policy { + case dmarc.PolicyReject: + code := 550 + enchCode := exterrors.EnhancedCode{5, 7, 1} + if dmarcRes.Authres.Value == authres.ResultTempError { + code = 450 + enchCode[0] = 4 + } + return &exterrors.SMTPError{ + Code: code, + EnhancedCode: enchCode, + Message: "DMARC check failed", + CheckName: "dmarc", + Misc: map[string]interface{}{ + "reason": dmarcRes.Authres.Reason, + "dkim_res": dmarcRes.DKIMResult.Value, + "dkim_domain": dmarcRes.DKIMResult.Domain, + "spf_res": dmarcRes.SPFResult.Value, + "spf_from": dmarcRes.SPFResult.From, + }, + } + case dmarc.PolicyQuarantine: + cr.msgMeta.Quarantine = true + + // Mimick the message structure for regular checks. + cr.log.Msg("quarantined", "reason", dmarcRes.Authres.Reason, "check", "dmarc") + } + } + + // After results for all checks are checked, authRes will be populated with values + // we should put into Authentication-Results header. + if len(cr.mergedRes.AuthResult) != 0 { + header.Add("Authentication-Results", authres.Format(hostname, cr.mergedRes.AuthResult)) + } + + for field := cr.mergedRes.Header.Fields(); field.Next(); { + formatted, err := field.Raw() + if err != nil { + cr.log.Error("malformed header field added by check", err) + } + header.AddRaw(formatted) + } + return nil +} + +func (cr *checkRunner) close() { + cr.dmarcVerify.Close() + for _, state := range cr.states { + state.Close() + } +} diff --git a/internal/msgpipeline/check_test.go b/internal/msgpipeline/check_test.go new file mode 100644 index 0000000..72fcb79 --- /dev/null +++ b/internal/msgpipeline/check_test.go @@ -0,0 +1,448 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "errors" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestMsgPipeline_Checks(t *testing.T) { + target := testutils.Target{} + check1, check2 := testutils.Check{}, testutils.Check{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&check1, &check2}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "whatever@whatever", []string{"whatever@whatever"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + if target.Messages[0].MsgMeta.Quarantine { + t.Fatalf("message is quarantined when it shouldn't") + } + + if check1.UnclosedStates != 0 || check2.UnclosedStates != 0 { + t.Fatalf("checks state objects leak or double-closed, alive counters: %v, %v", check1.UnclosedStates, check2.UnclosedStates) + } +} + +func TestMsgPipeline_AuthResults(t *testing.T) { + target := testutils.Target{} + check1, check2 := testutils.Check{ + BodyRes: module.CheckResult{ + AuthResult: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultFail, + From: "FROM", + Helo: "HELO", + }, + }, + }, + }, testutils.Check{ + BodyRes: module.CheckResult{ + AuthResult: []authres.Result{ + &authres.SPFResult{ + Value: authres.ResultFail, + From: "FROM2", + Helo: "HELO2", + }, + }, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&check1, &check2}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Hostname: "TEST-HOST", + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "whatever@whatever", []string{"whatever@whatever"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + authRes := target.Messages[0].Header.Get("Authentication-Results") + id, parsed, err := authres.Parse(authRes) + if err != nil { + t.Fatalf("failed to parse results") + } + if id != "TEST-HOST" { + t.Fatalf("wrong authres identifier") + } + if len(parsed) != 2 { + t.Fatalf("wrong amount of parts, want %d, got %d", 2, len(parsed)) + } + + var seen1, seen2 bool + for _, parts := range parsed { + spfPart, ok := parts.(*authres.SPFResult) + if !ok { + t.Fatalf("Not SPFResult") + } + + if spfPart.From == "FROM" { + seen1 = true + } + if spfPart.From == "FROM2" { + seen2 = true + } + } + + if !seen1 { + t.Fatalf("First authRes is missing") + } + if !seen2 { + t.Fatalf("Second authRes is missing") + } + + if check1.UnclosedStates != 0 || check2.UnclosedStates != 0 { + t.Fatalf("checks state objects leak or double-closed, alive counters: %v, %v", check1.UnclosedStates, check2.UnclosedStates) + } +} + +func TestMsgPipeline_Headers(t *testing.T) { + hdr1 := textproto.Header{} + hdr1.Add("HDR1", "1") + hdr2 := textproto.Header{} + hdr2.Add("HDR2", "2") + + target := testutils.Target{} + check1, check2 := testutils.Check{ + BodyRes: module.CheckResult{ + Header: hdr1, + }, + }, testutils.Check{ + BodyRes: module.CheckResult{ + Header: hdr2, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&check1, &check2}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Hostname: "TEST-HOST", + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "whatever@whatever", []string{"whatever@whatever"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + if target.Messages[0].Header.Get("HDR1") != "1" { + t.Fatalf("wrong HDR1 value, want %s, got %s", "1", target.Messages[0].Header.Get("HDR1")) + } + if target.Messages[0].Header.Get("HDR2") != "2" { + t.Fatalf("wrong HDR2 value, want %s, got %s", "1", target.Messages[0].Header.Get("HDR2")) + } + + if check1.UnclosedStates != 0 || check2.UnclosedStates != 0 { + t.Fatalf("checks state objects leak or double-closed, alive counters: %v, %v", check1.UnclosedStates, check2.UnclosedStates) + } +} + +func TestMsgPipeline_Globalcheck_Errors(t *testing.T) { + target := testutils.Target{} + check_ := testutils.Check{ + InitErr: errors.New("1"), + ConnRes: module.CheckResult{Reject: true, Reason: errors.New("2")}, + SenderRes: module.CheckResult{Reject: true, Reason: errors.New("3")}, + RcptRes: module.CheckResult{Reject: true, Reason: errors.New("4")}, + BodyRes: module.CheckResult{Reject: true, Reason: errors.New("5")}, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&check_}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Hostname: "TEST-HOST", + Log: testutils.Logger(t, "msgpipeline"), + } + + t.Run("init err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.InitErr = nil + + t.Run("conn err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.ConnRes.Reject = false + + t.Run("mail from err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.SenderRes.Reject = false + + t.Run("rcpt to err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.RcptRes.Reject = false + + t.Run("body err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.BodyRes.Reject = false + + t.Run("no err", func(t *testing.T) { + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + }) + + if check_.UnclosedStates != 0 { + t.Fatalf("check state objects leak or double-closed, counters: %d", check_.UnclosedStates) + } +} + +func TestMsgPipeline_SourceCheck_Errors(t *testing.T) { + target := testutils.Target{} + check_ := testutils.Check{ + InitErr: errors.New("1"), + ConnRes: module.CheckResult{Reject: true, Reason: errors.New("2")}, + SenderRes: module.CheckResult{Reject: true, Reason: errors.New("3")}, + RcptRes: module.CheckResult{Reject: true, Reason: errors.New("4")}, + BodyRes: module.CheckResult{Reject: true, Reason: errors.New("5")}, + } + globalCheck := testutils.Check{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&globalCheck}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + checks: []module.Check{&check_}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Hostname: "TEST-HOST", + Log: testutils.Logger(t, "msgpipeline"), + } + + t.Run("init err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.InitErr = nil + + t.Run("conn err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.ConnRes.Reject = false + + t.Run("mail from err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.SenderRes.Reject = false + + t.Run("rcpt to err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.RcptRes.Reject = false + + t.Run("body err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.BodyRes.Reject = false + + t.Run("no err", func(t *testing.T) { + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + }) + + if check_.UnclosedStates != 0 || globalCheck.UnclosedStates != 0 { + t.Fatalf("check state objects leak or double-closed, counters: %d, %d", + check_.UnclosedStates, globalCheck.UnclosedStates) + } +} + +func TestMsgPipeline_RcptCheck_Errors(t *testing.T) { + target := testutils.Target{} + check_ := testutils.Check{ + InitErr: errors.New("1"), + ConnRes: module.CheckResult{Reject: true, Reason: errors.New("2")}, + SenderRes: module.CheckResult{Reject: true, Reason: errors.New("3")}, + RcptRes: module.CheckResult{Reject: true, Reason: errors.New("4")}, + BodyRes: module.CheckResult{Reject: true, Reason: errors.New("5")}, + + InstName: "err_check", + } + // Added to check whether it leaks. + globalCheck := testutils.Check{InstName: "global_check"} + sourceCheck := testutils.Check{InstName: "source_check"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&globalCheck}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + checks: []module.Check{&check_}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Hostname: "TEST-HOST", + Log: testutils.Logger(t, "msgpipeline"), + } + + t.Run("init err", func(t *testing.T) { + d.Log = testutils.Logger(t, "msgpipeline") + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + + t.Log("!!!", check_.UnclosedStates) + }) + + check_.InitErr = nil + + t.Run("conn err", func(t *testing.T) { + d.Log = testutils.Logger(t, "msgpipeline") + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + + t.Log("!!!", check_.UnclosedStates) + }) + + check_.ConnRes.Reject = false + + t.Run("mail from err", func(t *testing.T) { + d.Log = testutils.Logger(t, "msgpipeline") + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + + t.Log("!!!", check_.UnclosedStates) + }) + + check_.SenderRes.Reject = false + + t.Run("rcpt to err", func(t *testing.T) { + d.Log = testutils.Logger(t, "msgpipeline") + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.RcptRes.Reject = false + + t.Run("body err", func(t *testing.T) { + d.Log = testutils.Logger(t, "msgpipeline") + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + check_.BodyRes.Reject = false + + t.Run("no err", func(t *testing.T) { + d.Log = testutils.Logger(t, "msgpipeline") + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + }) + + if check_.UnclosedStates != 0 || sourceCheck.UnclosedStates != 0 || globalCheck.UnclosedStates != 0 { + t.Fatalf("check state objects leak or double-closed, counters: %d, %d, %d", + check_.UnclosedStates, sourceCheck.UnclosedStates, globalCheck.UnclosedStates) + } +} diff --git a/internal/msgpipeline/config.go b/internal/msgpipeline/config.go new file mode 100644 index 0000000..46693e6 --- /dev/null +++ b/internal/msgpipeline/config.go @@ -0,0 +1,397 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "fmt" + "strconv" + "strings" + + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/modify" +) + +type sourceIn struct { + t module.Table + block sourceBlock +} + +type msgpipelineCfg struct { + globalChecks []module.Check + globalModifiers modify.Group + sourceIn []sourceIn + perSource map[string]sourceBlock + defaultSource sourceBlock + doDMARC bool +} + +func parseMsgPipelineRootCfg(globals map[string]interface{}, nodes []config.Node) (msgpipelineCfg, error) { + cfg := msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + } + var defaultSrcRaw []config.Node + var othersRaw []config.Node + for _, node := range nodes { + switch node.Name { + case "check": + globalChecks, err := parseChecksGroup(globals, node) + if err != nil { + return msgpipelineCfg{}, err + } + + cfg.globalChecks = append(cfg.globalChecks, globalChecks...) + case "modify": + globalModifiers, err := parseModifiersGroup(globals, node) + if err != nil { + return msgpipelineCfg{}, err + } + + cfg.globalModifiers.Modifiers = append(cfg.globalModifiers.Modifiers, globalModifiers.Modifiers...) + case "source_in": + var tbl module.Table + if err := modconfig.ModuleFromNode("table", node.Args, config.Node{}, globals, &tbl); err != nil { + return msgpipelineCfg{}, err + } + srcBlock, err := parseMsgPipelineSrcCfg(globals, node.Children) + if err != nil { + return msgpipelineCfg{}, err + } + cfg.sourceIn = append(cfg.sourceIn, sourceIn{ + t: tbl, + block: srcBlock, + }) + case "source": + srcBlock, err := parseMsgPipelineSrcCfg(globals, node.Children) + if err != nil { + return msgpipelineCfg{}, err + } + + if len(node.Args) == 0 { + return msgpipelineCfg{}, config.NodeErr(node, "expected at least one source matching rule") + } + + for _, rule := range node.Args { + if strings.Contains(rule, "@") { + rule, err = address.ForLookup(rule) + } else { + rule, err = dns.ForLookup(rule) + } + if err != nil { + return msgpipelineCfg{}, config.NodeErr(node, "invalid source match rule: %v: %v", rule, err) + } + + if !validMatchRule(rule) { + return msgpipelineCfg{}, config.NodeErr(node, "invalid source routing rule: %v", rule) + } + + if _, ok := cfg.perSource[rule]; ok { + continue + } + + cfg.perSource[rule] = srcBlock + } + case "default_source": + if defaultSrcRaw != nil { + return msgpipelineCfg{}, config.NodeErr(node, "duplicate 'default_source' block") + } + defaultSrcRaw = node.Children + case "dmarc": + switch len(node.Args) { + case 1: + switch node.Args[0] { + case "yes": + cfg.doDMARC = true + case "no": + default: + return msgpipelineCfg{}, config.NodeErr(node, "invalid argument for dmarc") + } + case 0: + cfg.doDMARC = true + } + case "deliver_to", "reroute", "destination_in", "destination", "default_destination", "reject": + othersRaw = append(othersRaw, node) + default: + return msgpipelineCfg{}, config.NodeErr(node, "unknown pipeline directive: %s", node.Name) + } + } + + if len(cfg.perSource) == 0 && len(defaultSrcRaw) == 0 { + if len(othersRaw) == 0 { + return msgpipelineCfg{}, fmt.Errorf("empty pipeline configuration, use 'reject' to reject messages") + } + + var err error + cfg.defaultSource, err = parseMsgPipelineSrcCfg(globals, othersRaw) + return cfg, err + } else if len(othersRaw) != 0 { + return msgpipelineCfg{}, config.NodeErr(othersRaw[0], "can't put handling directives together with source rules, did you mean to put it into 'default_source' block or into all source blocks?") + } + + if len(defaultSrcRaw) == 0 { + return msgpipelineCfg{}, config.NodeErr(nodes[0], "missing or empty default source block, use default_source { reject } to reject messages") + } + + var err error + cfg.defaultSource, err = parseMsgPipelineSrcCfg(globals, defaultSrcRaw) + return cfg, err +} + +func parseMsgPipelineSrcCfg(globals map[string]interface{}, nodes []config.Node) (sourceBlock, error) { + src := sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + } + var defaultRcptRaw []config.Node + var othersRaw []config.Node + for _, node := range nodes { + switch node.Name { + case "check": + checks, err := parseChecksGroup(globals, node) + if err != nil { + return sourceBlock{}, err + } + + src.checks = append(src.checks, checks...) + case "modify": + modifiers, err := parseModifiersGroup(globals, node) + if err != nil { + return sourceBlock{}, err + } + + src.modifiers.Modifiers = append(src.modifiers.Modifiers, modifiers.Modifiers...) + case "destination_in": + var tbl module.Table + if err := modconfig.ModuleFromNode("table", node.Args, config.Node{}, globals, &tbl); err != nil { + return sourceBlock{}, err + } + rcptBlock, err := parseMsgPipelineRcptCfg(globals, node.Children) + if err != nil { + return sourceBlock{}, err + } + src.rcptIn = append(src.rcptIn, rcptIn{ + t: tbl, + block: rcptBlock, + }) + case "destination": + rcptBlock, err := parseMsgPipelineRcptCfg(globals, node.Children) + if err != nil { + return sourceBlock{}, err + } + + if len(node.Args) == 0 { + return sourceBlock{}, config.NodeErr(node, "expected at least one destination match rule") + } + + for _, rule := range node.Args { + if strings.Contains(rule, "@") { + rule, err = address.ForLookup(rule) + } else { + rule, err = dns.ForLookup(rule) + } + if err != nil { + return sourceBlock{}, config.NodeErr(node, "invalid destination match rule: %v: %v", rule, err) + } + + if !validMatchRule(rule) { + return sourceBlock{}, config.NodeErr(node, "invalid destination match rule: %v", rule) + } + + if _, ok := src.perRcpt[rule]; ok { + continue + } + + src.perRcpt[rule] = rcptBlock + } + case "default_destination": + if defaultRcptRaw != nil { + return sourceBlock{}, config.NodeErr(node, "duplicate 'default_destination' block") + } + defaultRcptRaw = node.Children + case "deliver_to", "reroute", "reject": + othersRaw = append(othersRaw, node) + default: + return sourceBlock{}, config.NodeErr(node, "unknown pipeline directive: %s", node.Name) + } + } + + if len(src.perRcpt) == 0 && len(defaultRcptRaw) == 0 { + if len(othersRaw) == 0 { + return sourceBlock{}, fmt.Errorf("empty source block, use 'reject' to reject messages") + } + + var err error + src.defaultRcpt, err = parseMsgPipelineRcptCfg(globals, othersRaw) + return src, err + } else if len(othersRaw) != 0 { + return sourceBlock{}, config.NodeErr(othersRaw[0], "can't put handling directives together with destination rules, did you mean to put it into 'default' block or into all recipient blocks?") + } + + if len(defaultRcptRaw) == 0 { + return sourceBlock{}, config.NodeErr(nodes[0], "missing or empty default destination block, use default_destination { reject } to reject messages") + } + + var err error + src.defaultRcpt, err = parseMsgPipelineRcptCfg(globals, defaultRcptRaw) + return src, err +} + +func parseMsgPipelineRcptCfg(globals map[string]interface{}, nodes []config.Node) (*rcptBlock, error) { + rcpt := rcptBlock{} + for _, node := range nodes { + switch node.Name { + case "check": + checks, err := parseChecksGroup(globals, node) + if err != nil { + return nil, err + } + + rcpt.checks = append(rcpt.checks, checks...) + case "modify": + modifiers, err := parseModifiersGroup(globals, node) + if err != nil { + return nil, err + } + + rcpt.modifiers.Modifiers = append(rcpt.modifiers.Modifiers, modifiers.Modifiers...) + case "deliver_to": + if rcpt.rejectErr != nil { + return nil, config.NodeErr(node, "can't use 'reject' and 'deliver_to' together") + } + + if len(node.Args) == 0 { + return nil, config.NodeErr(node, "required at least one argument") + } + mod, err := modconfig.DeliveryTarget(globals, node.Args, node) + if err != nil { + return nil, err + } + + rcpt.targets = append(rcpt.targets, mod) + case "reroute": + if len(node.Children) == 0 { + return nil, config.NodeErr(node, "missing or empty reroute pipeline configuration") + } + + pipeline, err := New(globals, node.Children) + if err != nil { + return nil, err + } + + rcpt.targets = append(rcpt.targets, pipeline) + case "reject": + if len(rcpt.targets) != 0 { + return nil, config.NodeErr(node, "can't use 'reject' and 'deliver_to' together") + } + + var err error + rcpt.rejectErr, err = parseRejectDirective(node) + if err != nil { + return nil, err + } + default: + return nil, config.NodeErr(node, "invalid directive") + } + } + return &rcpt, nil +} + +func parseRejectDirective(node config.Node) (*exterrors.SMTPError, error) { + code := 554 + enchCode := exterrors.EnhancedCode{5, 7, 0} + msg := "Message rejected due to a local policy" + var err error + switch len(node.Args) { + case 3: + msg = node.Args[2] + if msg == "" { + return nil, config.NodeErr(node, "message can't be empty") + } + fallthrough + case 2: + enchCode, err = parseEnhancedCode(node.Args[1]) + if err != nil { + return nil, config.NodeErr(node, "%v", err) + } + if enchCode[0] != 4 && enchCode[0] != 5 { + return nil, config.NodeErr(node, "enhanced code should use either 4 or 5 as a first number") + } + fallthrough + case 1: + code, err = strconv.Atoi(node.Args[0]) + if err != nil { + return nil, config.NodeErr(node, "invalid error code integer: %v", err) + } + if (code/100) != 4 && (code/100) != 5 { + return nil, config.NodeErr(node, "error code should start with either 4 or 5") + } + case 0: + default: + return nil, config.NodeErr(node, "invalid count of arguments") + } + return &exterrors.SMTPError{ + Code: code, + EnhancedCode: enchCode, + Message: msg, + Reason: "reject directive used", + }, nil +} + +func parseEnhancedCode(s string) (exterrors.EnhancedCode, error) { + parts := strings.Split(s, ".") + if len(parts) != 3 { + return exterrors.EnhancedCode{}, fmt.Errorf("wrong amount of enhanced code parts") + } + + code := exterrors.EnhancedCode{} + for i, part := range parts { + num, err := strconv.Atoi(part) + if err != nil { + return code, err + } + code[i] = num + } + return code, nil +} + +func parseChecksGroup(globals map[string]interface{}, node config.Node) ([]module.Check, error) { + var cg *CheckGroup + err := modconfig.GroupFromNode("checks", node.Args, node, globals, &cg) + if err != nil { + return nil, err + } + return cg.L, nil +} + +func parseModifiersGroup(globals map[string]interface{}, node config.Node) (modify.Group, error) { + // Module object is *modify.Group, not modify.Group. + var mg *modify.Group + err := modconfig.GroupFromNode("modifiers", node.Args, node, globals, &mg) + if err != nil { + return modify.Group{}, err + } + return *mg, nil +} + +func validMatchRule(rule string) bool { + return address.ValidDomain(rule) || address.Valid(rule) +} diff --git a/internal/msgpipeline/config_test.go b/internal/msgpipeline/config_test.go new file mode 100644 index 0000000..24d7e51 --- /dev/null +++ b/internal/msgpipeline/config_test.go @@ -0,0 +1,440 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "reflect" + "strings" + "testing" + + parser "github.com/foxcpp/maddy/framework/cfgparser" + "github.com/foxcpp/maddy/framework/exterrors" +) + +func policyError(code int) error { + return &exterrors.SMTPError{ + Message: "Message rejected due to a local policy", + Code: code, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Reason: "reject directive used", + } +} + +func TestMsgPipelineCfg(t *testing.T) { + cases := []struct { + name string + str string + value msgpipelineCfg + fail bool + }{ + { + name: "basic", + str: ` + source example.com { + destination example.org { + reject 410 + } + default_destination { + reject 420 + } + } + default_source { + destination example.org { + reject 430 + } + default_destination { + reject 440 + } + }`, + value: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "example.com": { + perRcpt: map[string]*rcptBlock{ + "example.org": { + rejectErr: policyError(410), + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: policyError(420), + }, + }, + }, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "example.org": { + rejectErr: policyError(430), + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: policyError(440), + }, + }, + }, + }, + { + name: "implied default destination", + str: ` + source example.com { + reject 410 + } + default_source { + reject 420 + }`, + value: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + rejectErr: policyError(410), + }, + }, + }, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + rejectErr: policyError(420), + }, + }, + }, + }, + { + name: "implied default sender", + str: ` + destination example.com { + reject 410 + } + default_destination { + reject 420 + }`, + value: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "example.com": { + rejectErr: policyError(410), + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: policyError(420), + }, + }, + }, + }, + { + name: "missing default source handler", + str: ` + source example.org { + reject 410 + }`, + fail: true, + }, + { + name: "missing default destination handler", + str: ` + destination example.org { + reject 410 + }`, + fail: true, + }, + { + name: "invalid domain", + str: ` + destination .. { + reject 410 + } + default_destination { + reject 500 + }`, + fail: true, + }, + { + name: "invalid address", + str: ` + destination @example. { + reject 410 + } + default_destination { + reject 500 + }`, + fail: true, + }, + { + name: "invalid address", + str: ` + destination @example. { + reject 421 + } + default_destination { + reject 500 + }`, + fail: true, + }, + { + name: "invalid reject code", + str: ` + destination example.com { + reject 200 + } + default_destination { + reject 500 + }`, + fail: true, + }, + { + name: "destination together with source", + str: ` + destination example.com { + reject 410 + } + source example.org { + reject 420 + } + default_source { + reject 430 + }`, + fail: true, + }, + { + name: "empty destination rule", + str: ` + destination { + reject 410 + } + default_destination { + reject 420 + }`, + fail: true, + }, + } + + for _, case_ := range cases { + t.Run(case_.name, func(t *testing.T) { + cfg, _ := parser.Read(strings.NewReader(case_.str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil && !case_.fail { + t.Fatalf("unexpected parse error: %v", err) + } + if err == nil && case_.fail { + t.Fatalf("unexpected parse success") + } + if case_.fail { + t.Log(err) + return + } + if !reflect.DeepEqual(parsed, case_.value) { + t.Errorf("Wrong parsed configuration") + t.Errorf("Wanted: %+v", case_.value) + t.Errorf("Got: %+v", parsed) + } + }) + } +} + +func TestMsgPipelineCfg_SourceIn(t *testing.T) { + str := ` + source_in dummy { + deliver_to dummy + } + default_source { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.sourceIn) == 0 { + t.Fatalf("missing source_in dummy") + } +} + +func TestMsgPipelineCfg_DestIn(t *testing.T) { + str := ` + destination_in dummy { + deliver_to dummy + } + default_destination { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.defaultSource.rcptIn) == 0 { + t.Fatalf("missing destination_in dummy") + } +} + +func TestMsgPipelineCfg_GlobalChecks(t *testing.T) { + str := ` + check { + test_check + } + default_destination { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.globalChecks) == 0 { + t.Fatalf("missing test_check in globalChecks") + } +} + +func TestMsgPipelineCfg_GlobalChecksMultiple(t *testing.T) { + str := ` + check { + test_check + } + check { + test_check + } + default_destination { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.globalChecks) != 2 { + t.Fatalf("wrong amount of test_check's in globalChecks: %d", len(parsed.globalChecks)) + } +} + +func TestMsgPipelineCfg_SourceChecks(t *testing.T) { + str := ` + source example.org { + check { + test_check + } + + reject 500 + } + default_source { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.perSource["example.org"].checks) == 0 { + t.Fatalf("missing test_check in source checks") + } +} + +func TestMsgPipelineCfg_SourceChecks_Multiple(t *testing.T) { + str := ` + source example.org { + check { + test_check + } + check { + test_check + } + + reject 500 + } + default_source { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.perSource["example.org"].checks) != 2 { + t.Fatalf("wrong amount of test_check's in source checks: %d", len(parsed.perSource["example.org"].checks)) + } +} + +func TestMsgPipelineCfg_RcptChecks(t *testing.T) { + str := ` + destination example.org { + check { + test_check + } + + reject 500 + } + default_destination { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.defaultSource.perRcpt["example.org"].checks) == 0 { + t.Fatalf("missing test_check in rcpt checks") + } +} + +func TestMsgPipelineCfg_RcptChecks_Multiple(t *testing.T) { + str := ` + destination example.org { + check { + test_check + } + check { + test_check + } + + reject 500 + } + default_destination { + reject 500 + } + ` + + cfg, _ := parser.Read(strings.NewReader(str), "literal") + parsed, err := parseMsgPipelineRootCfg(nil, cfg) + if err != nil { + t.Fatalf("unexpected parse error: %v", err) + } + + if len(parsed.defaultSource.perRcpt["example.org"].checks) != 2 { + t.Fatalf("wrong amount of test_check's in rcpt checks: %d", len(parsed.defaultSource.perRcpt["example.org"].checks)) + } +} diff --git a/internal/msgpipeline/dmarc_test.go b/internal/msgpipeline/dmarc_test.go new file mode 100644 index 0000000..f942baf --- /dev/null +++ b/internal/msgpipeline/dmarc_test.go @@ -0,0 +1,224 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "bufio" + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "net" + "strings" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-msgauth/authres" + "github.com/emersion/go-smtp" + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func doTestDelivery(t *testing.T, tgt module.DeliveryTarget, from string, to []string, hdr string) (string, error) { + t.Helper() + + IDRaw := sha1.Sum([]byte(t.Name())) + encodedID := hex.EncodeToString(IDRaw[:]) + + body := buffer.MemoryBuffer{Slice: []byte("foobar")} + ctx := module.MsgMetadata{ + DontTraceSender: true, + ID: encodedID, + } + + hdrParsed, err := textproto.ReadHeader(bufio.NewReader(strings.NewReader(hdr))) + if err != nil { + panic(err) + } + + delivery, err := tgt.Start(context.Background(), &ctx, from) + if err != nil { + return encodedID, err + } + for _, rcpt := range to { + if err := delivery.AddRcpt(context.Background(), rcpt, smtp.RcptOptions{}); err != nil { + if err := delivery.Abort(context.Background()); err != nil { + t.Log("delivery.Abort:", err) + } + return encodedID, err + } + } + if err := delivery.Body(context.Background(), hdrParsed, body); err != nil { + if err := delivery.Abort(context.Background()); err != nil { + t.Log("delivery.Abort:", err) + } + return encodedID, err + } + if err := delivery.Commit(context.Background()); err != nil { + return encodedID, err + } + + return encodedID, err +} + +func dmarcResult(t *testing.T, hdr textproto.Header) authres.ResultValue { + field := hdr.Get("Authentication-Results") + if field == "" { + t.Fatalf("No results field") + } + + _, results, err := authres.Parse(field) + if err != nil { + t.Fatalf("Field parse err: %v", err) + } + + for _, res := range results { + dmarcRes, ok := res.(*authres.DMARCResult) + if ok { + return dmarcRes.Value + } + } + + t.Fatalf("No DMARC authres found") + return "" +} + +func TestDMARC(t *testing.T) { + test := func(zones map[string]mockdns.Zone, hdr string, authres []authres.Result, reject, quarantine bool, dmarcRes authres.ResultValue) { + t.Helper() + + tgt := testutils.Target{} + p := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{ + &testutils.Check{ + BodyRes: module.CheckResult{ + AuthResult: authres, + }, + }, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&tgt}, + }, + }, + doDMARC: true, + }, + Log: testutils.Logger(t, "pipeline"), + Resolver: &mockdns.Resolver{Zones: zones}, + } + + _, err := doTestDelivery(t, &p, "test@example.org", []string{"test@example.com"}, hdr) + if reject { + if err == nil { + t.Errorf("expected message to be rejected") + return + } + t.Log(err, exterrors.Fields(err)) + return + } + if err != nil { + t.Errorf("unexpected error: %v %+v", err, exterrors.Fields(err)) + return + } + + if len(tgt.Messages) != 1 { + t.Errorf("got %d messages", len(tgt.Messages)) + return + } + msg := tgt.Messages[0] + + if msg.MsgMeta.Quarantine != quarantine { + t.Errorf("msg.MsgMeta.Quarantine (%v) != quarantine (%v)", msg.MsgMeta.Quarantine, quarantine) + return + } + + res := dmarcResult(t, msg.Header) + if res != dmarcRes { + t.Errorf("expected DMARC result to be '%v', got '%v'", dmarcRes, res) + return + } + } + + // No policy => DMARC 'none' + test(map[string]mockdns.Zone{}, "From: hello@example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, false, false, authres.ResultNone) + + // Policy present & identifiers align => DMARC 'pass' + test(map[string]mockdns.Zone{ + "_dmarc.example.org.": { + TXT: []string{"v=DMARC1; p=none"}, + }, + }, "From: hello@example.org\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, false, false, authres.ResultPass) + + // Policy fetch error => DMARC 'permerror' but the message + // is accepted. + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + Err: errors.New("the dns server is going insane"), + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, false, false, authres.ResultPermError) + + // Policy fetch error => DMARC 'temperror' but the message + // is rejected ("fail closed") + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + Err: &net.DNSError{ + Err: "the dns server is going insane, temporary", + IsTemporary: true, + }, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, true, false, authres.ResultTempError) + + // Misaligned From vs DKIM => DMARC 'fail', policy says to reject + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + TXT: []string{"v=DMARC1; p=reject"}, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, true, false, "") + + // Misaligned From vs DKIM => DMARC 'fail', policy says to quarantine. + test(map[string]mockdns.Zone{ + "_dmarc.example.com.": { + TXT: []string{"v=DMARC1; p=quarantine"}, + }, + }, "From: hello@example.com\r\n\r\n", []authres.Result{ + &authres.DKIMResult{Value: authres.ResultPass, Domain: "example.org"}, + &authres.SPFResult{Value: authres.ResultNone, From: "example.org", Helo: "mx.example.org"}, + }, false, true, authres.ResultFail) +} diff --git a/internal/msgpipeline/metrics.go b/internal/msgpipeline/metrics.go new file mode 100644 index 0000000..691c047 --- /dev/null +++ b/internal/msgpipeline/metrics.go @@ -0,0 +1,47 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import "github.com/prometheus/client_golang/prometheus" + +var ( + checkReject = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "check", + Name: "reject", + Help: "Number of times a check returned 'reject' result (may be more than processed messages if check does so on per-recipient basis)", + }, + []string{"check"}, + ) + checkQuarantined = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "check", + Name: "quarantined", + Help: "Number of times a check returned 'quarantine' result (may be more than processed messages if check does so on per-recipient basis)", + }, + []string{"check"}, + ) +) + +func init() { + prometheus.MustRegister(checkReject) + prometheus.MustRegister(checkQuarantined) +} diff --git a/internal/msgpipeline/modifier_test.go b/internal/msgpipeline/modifier_test.go new file mode 100644 index 0000000..ac3a0d7 --- /dev/null +++ b/internal/msgpipeline/modifier_test.go @@ -0,0 +1,709 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "errors" + "testing" + + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/modify" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestMsgPipeline_SenderModifier(t *testing.T) { + target := testutils.Target{} + modifier := testutils.Modifier{ + InstName: "test_modifier", + MailFrom: map[string]string{ + "sender@example.com": "sender2@example.com", + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{modifier}, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender2@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if modifier.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", modifier.UnclosedStates) + } +} + +func TestMsgPipeline_SenderModifier_Multiple(t *testing.T) { + target := testutils.Target{} + mod1, mod2 := testutils.Modifier{ + InstName: "first_modifier", + MailFrom: map[string]string{ + "sender@example.com": "sender2@example.com", + }, + }, testutils.Modifier{ + InstName: "second_modifier", + MailFrom: map[string]string{ + "sender2@example.com": "sender3@example.com", + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{mod1, mod2}, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender3@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if mod1.UnclosedStates != 0 || mod2.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d, %d", mod1.UnclosedStates, mod2.UnclosedStates) + } +} + +func TestMsgPipeline_SenderModifier_PreDispatch(t *testing.T) { + target := testutils.Target{InstName: "target"} + mod := testutils.Modifier{ + InstName: "test_modifier", + MailFrom: map[string]string{ + "sender@example.com": "sender@example.org", + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{mod}, + }, + perSource: map[string]sourceBlock{ + "example.org": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + defaultSource: sourceBlock{rejectErr: errors.New("default src block used")}, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 1, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if mod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", mod.UnclosedStates) + } +} + +func TestMsgPipeline_SenderModifier_PostDispatch(t *testing.T) { + target := testutils.Target{InstName: "target"} + mod := testutils.Modifier{ + InstName: "test_modifier", + MailFrom: map[string]string{ + "sender@example.org": "sender@example.com", + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "example.org": { + modifiers: modify.Group{ + Modifiers: []module.Modifier{mod}, + }, + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + defaultSource: sourceBlock{rejectErr: errors.New("default src block used")}, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 1, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if mod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", mod.UnclosedStates) + } +} + +func TestMsgPipeline_SenderModifier_PerRcpt(t *testing.T) { + // Modifier below will be no-op due to implementation limitations. + + comTarget, orgTarget := testutils.Target{InstName: "com_target"}, testutils.Target{InstName: "org_target"} + mod := testutils.Modifier{ + InstName: "test_modifier", + MailFrom: map[string]string{ + "sender@example.com": "sender2@example.com", + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "example.com": { + modifiers: modify.Group{ + Modifiers: []module.Modifier{mod}, + }, + targets: []module.DeliveryTarget{&comTarget}, + }, + "example.org": { + modifiers: modify.Group{}, + targets: []module.DeliveryTarget{&orgTarget}, + }, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt@example.com", "rcpt@example.org"}) + + if len(comTarget.Messages) != 1 { + t.Fatalf("wrong amount of messages received for comTarget, want %d, got %d", 1, len(comTarget.Messages)) + } + testutils.CheckTestMessage(t, &comTarget, 0, "sender@example.com", []string{"rcpt@example.com"}) + + if len(orgTarget.Messages) != 1 { + t.Fatalf("wrong amount of messages received for orgTarget, want %d, got %d", 1, len(orgTarget.Messages)) + } + testutils.CheckTestMessage(t, &orgTarget, 0, "sender@example.com", []string{"rcpt@example.org"}) + + if mod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", mod.UnclosedStates) + } +} + +func TestMsgPipeline_RcptModifier(t *testing.T) { + target := testutils.Target{} + mod := testutils.Modifier{ + InstName: "test_modifier", + RcptTo: map[string][]string{ + "rcpt1@example.com": []string{"rcpt1-alias@example.com"}, + "rcpt2@example.com": []string{"rcpt2-alias@example.com"}, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{mod}, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1-alias@example.com", "rcpt2-alias@example.com"}) + + if mod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", mod.UnclosedStates) + } +} + +func TestMsgPipeline_RcptModifier_OriginalRcpt(t *testing.T) { + target := testutils.Target{} + mod := testutils.Modifier{ + InstName: "test_modifier", + RcptTo: map[string][]string{ + "rcpt1@example.com": []string{"rcpt1-alias@example.com"}, + "rcpt2@example.com": []string{"rcpt2-alias@example.com"}, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{mod}, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1-alias@example.com", "rcpt2-alias@example.com"}) + original1 := target.Messages[0].MsgMeta.OriginalRcpts["rcpt1-alias@example.com"] + if original1 != "rcpt1@example.com" { + t.Errorf("wrong OriginalRcpts value for first rcpt, want %s, got %s", "rcpt1@example.com", original1) + } + original2 := target.Messages[0].MsgMeta.OriginalRcpts["rcpt2-alias@example.com"] + if original2 != "rcpt2@example.com" { + t.Errorf("wrong OriginalRcpts value for first rcpt, want %s, got %s", "rcpt2@example.com", original2) + } + + if mod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", mod.UnclosedStates) + } +} + +func TestMsgPipeline_RcptModifier_OriginalRcpt_Multiple(t *testing.T) { + target := testutils.Target{} + mod1, mod2 := testutils.Modifier{ + InstName: "first_modifier", + RcptTo: map[string][]string{ + "rcpt1@example.com": []string{"rcpt1-alias@example.com"}, + "rcpt2@example.com": []string{"rcpt2-alias@example.com"}, + }, + }, testutils.Modifier{ + InstName: "second_modifier", + RcptTo: map[string][]string{ + "rcpt1-alias@example.com": []string{"rcpt1-alias2@example.com"}, + "rcpt2@example.com": []string{"wtf@example.com"}, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{mod1}, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + modifiers: modify.Group{ + Modifiers: []module.Modifier{mod2}, + }, + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1-alias2@example.com", "rcpt2-alias@example.com"}) + original1 := target.Messages[0].MsgMeta.OriginalRcpts["rcpt1-alias2@example.com"] + if original1 != "rcpt1@example.com" { + t.Errorf("wrong OriginalRcpts value for first rcpt, want %s, got %s", "rcpt1@example.com", original1) + } + original2 := target.Messages[0].MsgMeta.OriginalRcpts["rcpt2-alias@example.com"] + if original2 != "rcpt2@example.com" { + t.Errorf("wrong OriginalRcpts value for first rcpt, want %s, got %s", "rcpt2@example.com", original2) + } + + if mod1.UnclosedStates != 0 || mod2.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d, %d", mod1.UnclosedStates, mod2.UnclosedStates) + } +} + +func TestMsgPipeline_RcptModifier_Multiple(t *testing.T) { + target := testutils.Target{} + mod1, mod2 := testutils.Modifier{ + InstName: "first_modifier", + RcptTo: map[string][]string{ + "rcpt1@example.com": []string{"rcpt1-alias@example.com"}, + "rcpt2@example.com": []string{"rcpt2-alias@example.com"}, + }, + }, testutils.Modifier{ + InstName: "second_modifier", + RcptTo: map[string][]string{ + "rcpt1-alias@example.com": []string{"rcpt1-alias2@example.com"}, + "rcpt2@example.com": []string{"wtf@example.com"}, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{mod1, mod2}, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1-alias2@example.com", "rcpt2-alias@example.com"}) + + if mod1.UnclosedStates != 0 || mod2.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d, %d", mod1.UnclosedStates, mod2.UnclosedStates) + } +} + +func TestMsgPipeline_RcptModifier_PreDispatch(t *testing.T) { + target := testutils.Target{} + mod1, mod2 := testutils.Modifier{ + InstName: "first_modifier", + RcptTo: map[string][]string{ + "rcpt1@example.com": []string{"rcpt1-alias@example.com"}, + "rcpt2@example.com": []string{"rcpt2-alias@example.com"}, + }, + }, testutils.Modifier{ + InstName: "second_modifier", + RcptTo: map[string][]string{ + "rcpt1-alias@example.com": []string{"rcpt1-alias2@example.com"}, + "rcpt2@example.com": []string{"wtf@example.com"}, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{ + Modifiers: []module.Modifier{mod1}, + }, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + modifiers: modify.Group{Modifiers: []module.Modifier{mod2}}, + perRcpt: map[string]*rcptBlock{ + "rcpt2-alias@example.com": { + targets: []module.DeliveryTarget{&target}, + }, + "rcpt1-alias2@example.com": { + targets: []module.DeliveryTarget{&target}, + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("default rcpt is used"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1-alias2@example.com", "rcpt2-alias@example.com"}) + + if mod1.UnclosedStates != 0 || mod2.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d, %d", mod1.UnclosedStates, mod2.UnclosedStates) + } +} + +func TestMsgPipeline_RcptModifier_PostDispatch(t *testing.T) { + target := testutils.Target{} + mod := testutils.Modifier{ + InstName: "test_modifier", + RcptTo: map[string][]string{ + "rcpt1@example.com": []string{"rcpt1@example.org"}, + "rcpt2@example.com": []string{"rcpt2@example.org"}, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "example.com": { + modifiers: modify.Group{ + Modifiers: []module.Modifier{mod}, + }, + targets: []module.DeliveryTarget{&target}, + }, + "example.org": { + rejectErr: errors.New("wrong rcpt block is used"), + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("default rcpt is used"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1@example.org", "rcpt2@example.org"}) + + if mod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", mod.UnclosedStates) + } +} + +func TestMsgPipeline_GlobalModifier_Errors(t *testing.T) { + target := testutils.Target{} + mod := testutils.Modifier{ + InstName: "test_modifier", + InitErr: errors.New("1"), + MailFromErr: errors.New("2"), + RcptToErr: errors.New("3"), + BodyErr: errors.New("4"), + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalModifiers: modify.Group{Modifiers: []module.Modifier{&mod}}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + t.Run("init err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.InitErr = nil + + t.Run("mail from err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.MailFromErr = nil + + t.Run("rcpt to err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.RcptToErr = nil + + t.Run("body err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.BodyErr = nil + + t.Run("no err", func(t *testing.T) { + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + }) + + if mod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counter: %d", mod.UnclosedStates) + } +} + +func TestMsgPipeline_SourceModifier_Errors(t *testing.T) { + target := testutils.Target{} + mod := testutils.Modifier{ + InstName: "test_modifier", + InitErr: errors.New("1"), + MailFromErr: errors.New("2"), + RcptToErr: errors.New("3"), + BodyErr: errors.New("4"), + } + // Added to make sure it is freed properly too. + globalMod := testutils.Modifier{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + globalModifiers: modify.Group{Modifiers: []module.Modifier{&globalMod}}, + defaultSource: sourceBlock{ + modifiers: modify.Group{Modifiers: []module.Modifier{&mod}}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + t.Run("init err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.InitErr = nil + + t.Run("mail from err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.MailFromErr = nil + + t.Run("rcpt to err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.RcptToErr = nil + + t.Run("body err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.BodyErr = nil + + t.Run("no err", func(t *testing.T) { + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + }) + + if mod.UnclosedStates != 0 || globalMod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counters: %d, %d", + mod.UnclosedStates, globalMod.UnclosedStates) + } +} + +func TestMsgPipeline_RcptModifier_Errors(t *testing.T) { + target := testutils.Target{} + mod := testutils.Modifier{ + InstName: "test_modifier", + InitErr: errors.New("1"), + RcptToErr: errors.New("3"), + } + // Added to make sure it is freed properly too. + globalMod := testutils.Modifier{} + sourceMod := testutils.Modifier{} + + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + globalModifiers: modify.Group{Modifiers: []module.Modifier{&globalMod}}, + defaultSource: sourceBlock{ + modifiers: modify.Group{Modifiers: []module.Modifier{&sourceMod}}, + defaultRcpt: &rcptBlock{ + modifiers: modify.Group{Modifiers: []module.Modifier{&mod}}, + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + t.Run("init err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.InitErr = nil + + // MailFromErr test is inapplicable since RewriteSender is not called for per-rcpt + // modifiers. + + t.Run("rcpt to err", func(t *testing.T) { + _, err := testutils.DoTestDeliveryErr(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + if err == nil { + t.Fatal("expected error") + } + }) + + mod.RcptToErr = nil + + // BodyErr test is inapplicable since RewriteBody is not called for per-rcpt + // modifiers. + + t.Run("no err", func(t *testing.T) { + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + }) + + if mod.UnclosedStates != 0 || globalMod.UnclosedStates != 0 || sourceMod.UnclosedStates != 0 { + t.Fatalf("modifier state objects leak or double-closed, counters: %d, %d, %d", + mod.UnclosedStates, globalMod.UnclosedStates, sourceMod.UnclosedStates) + } +} diff --git a/internal/msgpipeline/module.go b/internal/msgpipeline/module.go new file mode 100644 index 0000000..cf30d22 --- /dev/null +++ b/internal/msgpipeline/module.go @@ -0,0 +1,70 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +type Module struct { + instName string + log log.Logger + *MsgPipeline +} + +func NewModule(modName, instName string, aliases, inlineArgs []string) (module.Module, error) { + return &Module{ + log: log.Logger{Name: "msgpipeline"}, + instName: instName, + }, nil +} + +func (m *Module) Init(cfg *config.Map) error { + var hostname string + cfg.String("hostname", true, true, "", &hostname) + cfg.Bool("debug", true, false, &m.log.Debug) + cfg.AllowUnknown() + other, err := cfg.Process() + if err != nil { + return err + } + + p, err := New(cfg.Globals, other) + if err != nil { + return err + } + m.MsgPipeline = p + m.MsgPipeline.Log = m.log + + return nil +} + +func (m *Module) Name() string { + return "msgpipeline" +} + +func (m *Module) InstanceName() string { + return m.instName +} + +func init() { + module.Register("msgpipeline", NewModule) +} diff --git a/internal/msgpipeline/msgpipeline.go b/internal/msgpipeline/msgpipeline.go new file mode 100644 index 0000000..388caab --- /dev/null +++ b/internal/msgpipeline/msgpipeline.go @@ -0,0 +1,655 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/modify" + "github.com/foxcpp/maddy/internal/target" + "golang.org/x/sync/errgroup" +) + +// MsgPipeline is a object that is responsible for selecting delivery targets +// for the message and running necessary checks and modifiers. +// +// It implements module.DeliveryTarget. +// +// It is not a "module object" and is intended to be used as part of message +// source (Submission, SMTP, JMAP modules) implementation. +type MsgPipeline struct { + msgpipelineCfg + Hostname string + Resolver dns.Resolver + + // Used to indicate the pipeline is handling messages received from the + // external source and not from any other module. That is, this MsgPipeline + // is an instance embedded in endpoint/smtp implementation, for example. + // + // This is a hack since only MsgPipeline can execute some operations at the + // right time but it is not a good idea to execute them multiple multiple + // times for a single message that might be actually handled my multiple + // pipelines via 'msgpipeline' module or 'reroute' directive. + // + // At the moment, the only such operation is the addition of the Received + // header field. See where it happens for explanation on why it is done + // exactly in this place. + FirstPipeline bool + + Log log.Logger +} + +type rcptIn struct { + t module.Table + block *rcptBlock +} + +type sourceBlock struct { + checks []module.Check + modifiers modify.Group + rejectErr error + rcptIn []rcptIn + perRcpt map[string]*rcptBlock + defaultRcpt *rcptBlock +} + +type rcptBlock struct { + checks []module.Check + modifiers modify.Group + rejectErr error + targets []module.DeliveryTarget +} + +func New(globals map[string]interface{}, cfg []config.Node) (*MsgPipeline, error) { + parsedCfg, err := parseMsgPipelineRootCfg(globals, cfg) + return &MsgPipeline{ + msgpipelineCfg: parsedCfg, + Resolver: dns.DefaultResolver(), + }, err +} + +func (d *MsgPipeline) RunEarlyChecks(ctx context.Context, state *module.ConnState) error { + eg, checkCtx := errgroup.WithContext(ctx) + + // TODO: See if there is some point in parallelization of this + // function. + for _, check := range d.globalChecks { + earlyCheck, ok := check.(module.EarlyCheck) + if !ok { + continue + } + + eg.Go(func() error { + return earlyCheck.CheckConnection(checkCtx, state) + }) + } + return eg.Wait() +} + +// Start starts new message delivery, runs connection and sender checks, sender modifiers +// and selects source block from config to use for handling. +// +// Returned module.Delivery implements PartialDelivery. If underlying target doesn't +// support it, msgpipeline will copy the returned error for all recipients handled +// by target. +func (d *MsgPipeline) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + dd := msgpipelineDelivery{ + d: d, + rcptModifiersState: make(map[*rcptBlock]module.ModifierState), + deliveries: make(map[module.DeliveryTarget]*delivery), + msgMeta: msgMeta, + log: target.DeliveryLogger(d.Log, msgMeta), + } + dd.checkRunner = newCheckRunner(msgMeta, dd.log, d.Resolver) + dd.checkRunner.doDMARC = d.doDMARC + + if msgMeta.OriginalRcpts == nil { + msgMeta.OriginalRcpts = map[string]string{} + } + + if err := dd.start(ctx, msgMeta, mailFrom); err != nil { + dd.close() + return nil, err + } + + return &dd, nil +} + +func (dd *msgpipelineDelivery) start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) error { + var err error + + if err := dd.checkRunner.checkConnSender(ctx, dd.d.globalChecks, mailFrom); err != nil { + return err + } + + if mailFrom, err = dd.initRunGlobalModifiers(ctx, msgMeta, mailFrom); err != nil { + return err + } + + sourceBlock, err := dd.srcBlockForAddr(ctx, mailFrom) + if err != nil { + return err + } + if sourceBlock.rejectErr != nil { + dd.log.Debugf("sender %s rejected with error: %v", mailFrom, sourceBlock.rejectErr) + return sourceBlock.rejectErr + } + dd.sourceBlock = sourceBlock + + if err := dd.checkRunner.checkConnSender(ctx, sourceBlock.checks, mailFrom); err != nil { + return err + } + + sourceModifiersState, err := sourceBlock.modifiers.ModStateForMsg(ctx, msgMeta) + if err != nil { + return err + } + mailFrom, err = sourceModifiersState.RewriteSender(ctx, mailFrom) + if err != nil { + return err + } + dd.sourceModifiersState = sourceModifiersState + + dd.sourceAddr = mailFrom + return nil +} + +func (dd *msgpipelineDelivery) initRunGlobalModifiers(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (string, error) { + globalModifiersState, err := dd.d.globalModifiers.ModStateForMsg(ctx, msgMeta) + if err != nil { + return "", err + } + mailFrom, err = globalModifiersState.RewriteSender(ctx, mailFrom) + if err != nil { + globalModifiersState.Close() + return "", err + } + dd.globalModifiersState = globalModifiersState + return mailFrom, nil +} + +func (dd *msgpipelineDelivery) srcBlockForAddr(ctx context.Context, mailFrom string) (sourceBlock, error) { + cleanFrom := mailFrom + if mailFrom != "" { + var err error + cleanFrom, err = address.ForLookup(mailFrom) + if err != nil { + return sourceBlock{}, &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 7}, + Message: "Unable to normalize the sender address", + Err: err, + } + } + } + + for _, srcIn := range dd.d.sourceIn { + _, ok, err := srcIn.t.Lookup(ctx, cleanFrom) + if err != nil { + dd.log.Error("source_in lookup failed", err, "key", cleanFrom) + continue + } + if !ok { + continue + } + return srcIn.block, nil + } + + // First try to match against complete address. + srcBlock, ok := dd.d.perSource[cleanFrom] + if !ok { + // Then try domain-only. + _, domain, err := address.Split(cleanFrom) + // mailFrom != "" is added as a special condition + // instead of extending address.Split because "" + // is not a valid RFC 282 address and only a special + // value for SMTP. + if err != nil && cleanFrom != "" { + return sourceBlock{}, &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 3}, + Message: "Invalid sender address", + Err: err, + Reason: "Can't extract local-part and host-part", + } + } + + // domain is already case-folded and normalized by the message source. + srcBlock, ok = dd.d.perSource[domain] + if !ok { + // Fallback to the default source block. + srcBlock = dd.d.defaultSource + dd.log.Debugf("sender %s matched by default rule", mailFrom) + } else { + dd.log.Debugf("sender %s matched by domain rule '%s'", mailFrom, domain) + } + } else { + dd.log.Debugf("sender %s matched by address rule '%s'", mailFrom, cleanFrom) + } + return srcBlock, nil +} + +type delivery struct { + module.Delivery + // Recipient addresses this delivery object is used for, original values (not modified by RewriteRcpt). + recipients []string +} + +type msgpipelineDelivery struct { + d *MsgPipeline + + globalModifiersState module.ModifierState + sourceModifiersState module.ModifierState + rcptModifiersState map[*rcptBlock]module.ModifierState + + log log.Logger + + sourceAddr string + sourceBlock sourceBlock + + deliveries map[module.DeliveryTarget]*delivery + msgMeta *module.MsgMetadata + checkRunner *checkRunner +} + +func (dd *msgpipelineDelivery) AddRcpt(ctx context.Context, to string, opts smtp.RcptOptions) error { + if err := dd.checkRunner.checkRcpt(ctx, dd.d.globalChecks, to); err != nil { + return err + } + if err := dd.checkRunner.checkRcpt(ctx, dd.sourceBlock.checks, to); err != nil { + return err + } + + originalTo := to + + newTo, err := dd.globalModifiersState.RewriteRcpt(ctx, to) + if err != nil { + return err + } + dd.log.Debugln("global rcpt modifiers:", to, "=>", newTo) + resultTo := newTo + newTo = []string{} + + for _, to = range resultTo { + var tempTo []string + tempTo, err = dd.sourceModifiersState.RewriteRcpt(ctx, to) + if err != nil { + return err + } + newTo = append(newTo, tempTo...) + } + dd.log.Debugln("per-source rcpt modifiers:", to, "=>", newTo) + resultTo = newTo + + for _, to = range resultTo { + wrapErr := func(err error) error { + return exterrors.WithFields(err, map[string]interface{}{ + "effective_rcpt": to, + }) + } + + rcptBlock, err := dd.rcptBlockForAddr(ctx, to) + if err != nil { + return wrapErr(err) + } + + if rcptBlock.rejectErr != nil { + return wrapErr(rcptBlock.rejectErr) + } + + if err := dd.checkRunner.checkRcpt(ctx, rcptBlock.checks, to); err != nil { + return wrapErr(err) + } + + rcptModifiersState, err := dd.getRcptModifiers(ctx, rcptBlock, to) + if err != nil { + return wrapErr(err) + } + + newTo, err = rcptModifiersState.RewriteRcpt(ctx, to) + if err != nil { + rcptModifiersState.Close() + return wrapErr(err) + } + dd.log.Debugln("per-rcpt modifiers:", to, "=>", newTo) + + for _, to = range newTo { + wrapErr = func(err error) error { + return exterrors.WithFields(err, map[string]interface{}{ + "effective_rcpt": to, + }) + } + + if originalTo != to { + dd.msgMeta.OriginalRcpts[to] = originalTo + } + + for _, tgt := range rcptBlock.targets { + // Do not wrap errors coming from nested pipeline target delivery since + // that pipeline itself will insert effective_rcpt field and could do + // its own rewriting - we do not want to hide it from the admin in + // error messages. + wrapErr := wrapErr + if _, ok := tgt.(*MsgPipeline); ok { + wrapErr = func(err error) error { return err } + } + + delivery, err := dd.getDelivery(ctx, tgt) + if err != nil { + return wrapErr(err) + } + + if err := delivery.AddRcpt(ctx, to, opts); err != nil { + return wrapErr(err) + } + delivery.recipients = append(delivery.recipients, originalTo) + } + } + } + + return nil +} + +func (dd *msgpipelineDelivery) Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error { + if err := dd.checkRunner.checkBody(ctx, dd.d.globalChecks, header, body); err != nil { + return err + } + if err := dd.checkRunner.checkBody(ctx, dd.sourceBlock.checks, header, body); err != nil { + return err + } + for blk := range dd.rcptModifiersState { + if err := dd.checkRunner.checkBody(ctx, blk.checks, header, body); err != nil { + return err + } + } + + if dd.d.FirstPipeline { + // Add Received *after* checks to make sure they see the message literally + // how we received it BUT place it below any other field that might be + // added by applyResults (including Authentication-Results) + // per recommendation in RFC 7001, Section 4 (see GH issue #135). + received, err := target.GenerateReceived(ctx, dd.msgMeta, dd.d.Hostname, dd.msgMeta.OriginalFrom) + if err != nil { + return err + } + header.Add("Received", received) + } + + if err := dd.checkRunner.applyResults(dd.d.Hostname, &header); err != nil { + return err + } + + // Run modifiers after Authentication-Results addition to make + // sure signatures, etc will cover it. + if err := dd.globalModifiersState.RewriteBody(ctx, &header, body); err != nil { + return err + } + if err := dd.sourceModifiersState.RewriteBody(ctx, &header, body); err != nil { + return err + } + for _, modifiers := range dd.rcptModifiersState { + if err := modifiers.RewriteBody(ctx, &header, body); err != nil { + return err + } + } + + for _, delivery := range dd.deliveries { + if err := delivery.Body(ctx, header, body); err != nil { + return err + } + dd.log.Debugf("delivery.Body ok, Delivery object = %T", delivery) + } + return nil +} + +// statusCollector wraps StatusCollector and adds reverse translation +// of recipients for all statuses.] +// +// We can't let delivery targets set statuses directly because they see +// modified addresses (RewriteRcpt) and we are supposed to report +// statuses using original values. Additionally, we should still avoid +// collect-and-them-report approach since statuses should be reported +// as soon as possible (that is required by LMTP). +type statusCollector struct { + originalRcpts map[string]string + wrapped module.StatusCollector +} + +func (sc statusCollector) SetStatus(rcptTo string, err error) { + original, ok := sc.originalRcpts[rcptTo] + if ok { + rcptTo = original + } + sc.wrapped.SetStatus(rcptTo, err) +} + +func (dd *msgpipelineDelivery) BodyNonAtomic(ctx context.Context, c module.StatusCollector, header textproto.Header, body buffer.Buffer) { + setStatusAll := func(err error) { + for _, delivery := range dd.deliveries { + for _, rcpt := range delivery.recipients { + c.SetStatus(rcpt, err) + } + } + } + + if err := dd.checkRunner.checkBody(ctx, dd.d.globalChecks, header, body); err != nil { + setStatusAll(err) + return + } + if err := dd.checkRunner.checkBody(ctx, dd.sourceBlock.checks, header, body); err != nil { + setStatusAll(err) + return + } + + // Run modifiers after Authentication-Results addition to make + // sure signatures, etc will cover it. + if err := dd.globalModifiersState.RewriteBody(ctx, &header, body); err != nil { + setStatusAll(err) + return + } + if err := dd.sourceModifiersState.RewriteBody(ctx, &header, body); err != nil { + setStatusAll(err) + return + } + for _, modifiers := range dd.rcptModifiersState { + if err := modifiers.RewriteBody(ctx, &header, body); err != nil { + setStatusAll(err) + return + } + } + + for _, delivery := range dd.deliveries { + partDelivery, ok := delivery.Delivery.(module.PartialDelivery) + if ok { + partDelivery.BodyNonAtomic(ctx, statusCollector{ + originalRcpts: dd.msgMeta.OriginalRcpts, + wrapped: c, + }, header, body) + continue + } + + if err := delivery.Body(ctx, header, body); err != nil { + for _, rcpt := range delivery.recipients { + c.SetStatus(rcpt, err) + } + } + } +} + +func (dd msgpipelineDelivery) Commit(ctx context.Context) error { + dd.close() + + for _, delivery := range dd.deliveries { + if err := delivery.Commit(ctx); err != nil { + // No point in Committing remaining deliveries, everything is broken already. + return err + } + } + return nil +} + +func (dd *msgpipelineDelivery) close() { + dd.checkRunner.close() + + if dd.globalModifiersState != nil { + dd.globalModifiersState.Close() + } + if dd.sourceModifiersState != nil { + dd.sourceModifiersState.Close() + } + for _, modifiers := range dd.rcptModifiersState { + modifiers.Close() + } +} + +func (dd msgpipelineDelivery) Abort(ctx context.Context) error { + dd.close() + + var lastErr error + for _, delivery := range dd.deliveries { + if err := delivery.Abort(ctx); err != nil { + dd.log.Debugf("delivery.Abort failure, Delivery object = %T: %v", delivery, err) + lastErr = err + // Continue anyway and try to Abort all remaining delivery objects. + } + } + return lastErr +} + +func (dd *msgpipelineDelivery) rcptBlockForAddr(ctx context.Context, rcptTo string) (*rcptBlock, error) { + cleanRcpt, err := address.ForLookup(rcptTo) + if err != nil { + return nil, &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 1, 2}, + Message: "Unable to normalize the recipient address", + Err: err, + } + } + + for _, rcptIn := range dd.sourceBlock.rcptIn { + _, ok, err := rcptIn.t.Lookup(ctx, cleanRcpt) + if err != nil { + dd.log.Error("destination_in lookup failed", err, "key", cleanRcpt) + continue + } + if !ok { + continue + } + return rcptIn.block, nil + } + + // First try to match against complete address. + rcptBlock, ok := dd.sourceBlock.perRcpt[cleanRcpt] + if !ok { + // Then try domain-only. + _, domain, err := address.Split(cleanRcpt) + if err != nil { + return nil, &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 3}, + Message: "Invalid recipient address", + Err: err, + Reason: "Can't extract local-part and host-part", + } + } + + // domain is already case-folded and normalized because it is a part of + // cleanRcpt. + rcptBlock, ok = dd.sourceBlock.perRcpt[domain] + if !ok { + // Fallback to the default source block. + rcptBlock = dd.sourceBlock.defaultRcpt + dd.log.Debugf("recipient %s matched by default rule (clean = %s)", rcptTo, cleanRcpt) + } else { + dd.log.Debugf("recipient %s matched by domain rule '%s'", rcptTo, domain) + } + } else { + dd.log.Debugf("recipient %s matched by address rule '%s'", rcptTo, cleanRcpt) + } + return rcptBlock, nil +} + +func (dd *msgpipelineDelivery) getRcptModifiers(ctx context.Context, rcptBlock *rcptBlock, rcptTo string) (module.ModifierState, error) { + rcptModifiersState, ok := dd.rcptModifiersState[rcptBlock] + if ok { + return rcptModifiersState, nil + } + + rcptModifiersState, err := rcptBlock.modifiers.ModStateForMsg(ctx, dd.msgMeta) + if err != nil { + return nil, err + } + + newSender, err := rcptModifiersState.RewriteSender(ctx, dd.sourceAddr) + if err == nil && newSender != dd.sourceAddr { + dd.log.Msg("Per-recipient modifier changed sender address. This is not supported and will "+ + "be ignored.", "rcpt", rcptTo, "originalFrom", dd.sourceAddr, "modifiedFrom", newSender) + } + + dd.rcptModifiersState[rcptBlock] = rcptModifiersState + return rcptModifiersState, nil +} + +func (dd *msgpipelineDelivery) getDelivery(ctx context.Context, tgt module.DeliveryTarget) (*delivery, error) { + delivery_, ok := dd.deliveries[tgt] + if ok { + return delivery_, nil + } + + deliveryObj, err := tgt.Start(ctx, dd.msgMeta, dd.sourceAddr) + if err != nil { + dd.log.Debugf("tgt.Start(%s) failure, target = %s: %v", dd.sourceAddr, objectName(tgt), err) + return nil, err + } + delivery_ = &delivery{Delivery: deliveryObj} + + dd.log.Debugf("tgt.Start(%s) ok, target = %s", dd.sourceAddr, objectName(tgt)) + + dd.deliveries[tgt] = delivery_ + return delivery_, nil +} + +// Mock returns a MsgPipeline that merely delivers messages to a specified target +// and runs a set of checks. +// +// It is meant for use in tests for modules that embed a pipeline object. +func Mock(tgt module.DeliveryTarget, globalChecks []module.Check) *MsgPipeline { + return &MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: globalChecks, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{tgt}, + }, + }, + }, + } +} diff --git a/internal/msgpipeline/msgpipeline_test.go b/internal/msgpipeline/msgpipeline_test.go new file mode 100644 index 0000000..9899eb0 --- /dev/null +++ b/internal/msgpipeline/msgpipeline_test.go @@ -0,0 +1,706 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "context" + "errors" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/modify" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestMsgPipeline_AllToTarget(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received, want %d, got %d", 1, len(target.Messages)) + } + + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) +} + +func TestMsgPipeline_PerSourceDomainSplit(t *testing.T) { + orgTarget, comTarget := testutils.Target{InstName: "orgTarget"}, testutils.Target{InstName: "comTarget"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&comTarget}, + }, + }, + "example.org": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&orgTarget}, + }, + }, + }, + defaultSource: sourceBlock{rejectErr: errors.New("default src block used")}, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + testutils.DoTestDelivery(t, &d, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(comTarget.Messages) != 1 { + t.Fatalf("wrong amount of messages received for comTarget, want %d, got %d", 1, len(comTarget.Messages)) + } + testutils.CheckTestMessage(t, &comTarget, 0, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(orgTarget.Messages) != 1 { + t.Fatalf("wrong amount of messages received for orgTarget, want %d, got %d", 1, len(orgTarget.Messages)) + } + testutils.CheckTestMessage(t, &orgTarget, 0, "sender@example.org", []string{"rcpt1@example.com", "rcpt2@example.com"}) +} + +func TestMsgPipeline_SourceIn(t *testing.T) { + tblTarget, comTarget := testutils.Target{InstName: "tblTarget"}, testutils.Target{InstName: "comTarget"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + sourceIn: []sourceIn{ + { + t: testutils.Table{}, + block: sourceBlock{rejectErr: errors.New("non-matching block was used")}, + }, + { + t: testutils.Table{Err: errors.New("this one will fail")}, + block: sourceBlock{rejectErr: errors.New("failing block was used")}, + }, + { + t: testutils.Table{ + M: map[string]string{ + "specific@example.com": "", + }, + }, + block: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&tblTarget}, + }, + }, + }, + }, + perSource: map[string]sourceBlock{ + "example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&comTarget}, + }, + }, + }, + defaultSource: sourceBlock{rejectErr: errors.New("default src block used")}, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt@example.com"}) + testutils.DoTestDelivery(t, &d, "specific@example.com", []string{"rcpt@example.com"}) + + if len(comTarget.Messages) != 1 { + t.Fatalf("wrong amount of messages received for comTarget, want %d, got %d", 1, len(comTarget.Messages)) + } + testutils.CheckTestMessage(t, &comTarget, 0, "sender@example.com", []string{"rcpt@example.com"}) + + if len(tblTarget.Messages) != 1 { + t.Fatalf("wrong amount of messages received for orgTarget, want %d, got %d", 1, len(tblTarget.Messages)) + } + testutils.CheckTestMessage(t, &tblTarget, 0, "specific@example.com", []string{"rcpt@example.com"}) +} + +func TestMsgPipeline_EmptyMAILFROM(t *testing.T) { + target := testutils.Target{InstName: "target"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 1, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "", []string{"rcpt1@example.com", "rcpt2@example.com"}) +} + +func TestMsgPipeline_EmptyMAILFROM_ExplicitDest(t *testing.T) { + target := testutils.Target{InstName: "target"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + defaultSource: sourceBlock{rejectErr: errors.New("default src block used")}, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "", []string{"rcpt1@example.com", "rcpt2@example.com"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 1, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "", []string{"rcpt1@example.com", "rcpt2@example.com"}) +} + +func TestMsgPipeline_PerRcptAddrSplit(t *testing.T) { + target1, target2 := testutils.Target{InstName: "target1"}, testutils.Target{InstName: "target2"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "rcpt1@example.com": { + targets: []module.DeliveryTarget{&target1}, + }, + "rcpt2@example.com": { + targets: []module.DeliveryTarget{&target2}, + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("defaultRcpt block used"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com"}) + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt2@example.com"}) + + if len(target1.Messages) != 1 { + t.Errorf("wrong amount of messages received for target1, want %d, got %d", 1, len(target1.Messages)) + } + testutils.CheckTestMessage(t, &target1, 0, "sender@example.com", []string{"rcpt1@example.com"}) + + if len(target2.Messages) != 1 { + t.Errorf("wrong amount of messages received for target1, want %d, got %d", 1, len(target2.Messages)) + } + testutils.CheckTestMessage(t, &target2, 0, "sender@example.com", []string{"rcpt2@example.com"}) +} + +func TestMsgPipeline_PerRcptDomainSplit(t *testing.T) { + target1, target2 := testutils.Target{InstName: "target1"}, testutils.Target{InstName: "target2"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "example.com": { + targets: []module.DeliveryTarget{&target1}, + }, + "example.org": { + targets: []module.DeliveryTarget{&target2}, + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("defaultRcpt block used"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "rcpt2@example.org"}) + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.org", "rcpt2@example.com"}) + + if len(target1.Messages) != 2 { + t.Errorf("wrong amount of messages received for target1, want %d, got %d", 2, len(target1.Messages)) + } + testutils.CheckTestMessage(t, &target1, 0, "sender@example.com", []string{"rcpt1@example.com"}) + testutils.CheckTestMessage(t, &target1, 1, "sender@example.com", []string{"rcpt2@example.com"}) + + if len(target2.Messages) != 2 { + t.Errorf("wrong amount of messages received for target2, want %d, got %d", 2, len(target2.Messages)) + } + testutils.CheckTestMessage(t, &target2, 0, "sender@example.com", []string{"rcpt2@example.org"}) + testutils.CheckTestMessage(t, &target2, 1, "sender@example.com", []string{"rcpt1@example.org"}) +} + +func TestMsgPipeline_DestInSplit(t *testing.T) { + target1, target2 := testutils.Target{InstName: "target1"}, testutils.Target{InstName: "target2"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + rcptIn: []rcptIn{ + { + t: testutils.Table{}, + block: &rcptBlock{rejectErr: errors.New("non-matching block was used")}, + }, + { + t: testutils.Table{Err: errors.New("nope")}, + block: &rcptBlock{rejectErr: errors.New("failing block was used")}, + }, + { + t: testutils.Table{ + M: map[string]string{ + "specific@example.com": "", + }, + }, + block: &rcptBlock{ + targets: []module.DeliveryTarget{&target2}, + }, + }, + }, + perRcpt: map[string]*rcptBlock{ + "example.com": { + targets: []module.DeliveryTarget{&target1}, + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("defaultRcpt block used"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt1@example.com", "specific@example.com"}) + + if len(target1.Messages) != 1 { + t.Errorf("wrong amount of messages received for target1, want %d, got %d", 1, len(target1.Messages)) + } + testutils.CheckTestMessage(t, &target1, 0, "sender@example.com", []string{"rcpt1@example.com"}) + + if len(target2.Messages) != 1 { + t.Errorf("wrong amount of messages received for target2, want %d, got %d", 1, len(target2.Messages)) + } + testutils.CheckTestMessage(t, &target2, 0, "sender@example.com", []string{"specific@example.com"}) +} + +func TestMsgPipeline_PerSourceAddrAndDomainSplit(t *testing.T) { + target1, target2 := testutils.Target{InstName: "target1"}, testutils.Target{InstName: "target2"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "sender1@example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target1}, + }, + }, + "example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target2}, + }, + }, + }, + defaultSource: sourceBlock{rejectErr: errors.New("default src block used")}, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender1@example.com", []string{"rcpt@example.com"}) + testutils.DoTestDelivery(t, &d, "sender2@example.com", []string{"rcpt@example.com"}) + + if len(target1.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target1, want %d, got %d", 1, len(target1.Messages)) + } + testutils.CheckTestMessage(t, &target1, 0, "sender1@example.com", []string{"rcpt@example.com"}) + + if len(target2.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target2, want %d, got %d", 1, len(target2.Messages)) + } + testutils.CheckTestMessage(t, &target2, 0, "sender2@example.com", []string{"rcpt@example.com"}) +} + +func TestMsgPipeline_PerSourceReject(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "sender1@example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + "example.com": { + perRcpt: map[string]*rcptBlock{}, + rejectErr: errors.New("go away"), + }, + }, + defaultSource: sourceBlock{rejectErr: errors.New("go away")}, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender1@example.com", []string{"rcpt@example.com"}) + + _, err := d.Start(context.Background(), &module.MsgMetadata{ID: "testing"}, "sender2@example.com") + if err == nil { + t.Error("expected error for delivery.Start, got nil") + } + + _, err = d.Start(context.Background(), &module.MsgMetadata{ID: "testing"}, "sender2@example.org") + if err == nil { + t.Error("expected error for delivery.Start, got nil") + } +} + +func TestMsgPipeline_PerRcptReject(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "rcpt1@example.com": { + targets: []module.DeliveryTarget{&target}, + }, + "example.com": { + rejectErr: errors.New("go away"), + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("go away"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + delivery, err := d.Start(context.Background(), &module.MsgMetadata{ID: "testing"}, "sender@example.com") + if err != nil { + t.Fatalf("unexpected Start err: %v", err) + } + defer func() { + if err := delivery.Abort(context.Background()); err != nil { + t.Fatalf("unexpected Abort err: %v", err) + } + }() + + if err := delivery.AddRcpt(context.Background(), "rcpt2@example.com", smtp.RcptOptions{}); err == nil { + t.Fatalf("expected error for delivery.AddRcpt(rcpt2@example.com), got nil") + } + if err := delivery.AddRcpt(context.Background(), "rcpt1@example.com", smtp.RcptOptions{}); err != nil { + t.Fatalf("unexpected AddRcpt err for %s: %v", "rcpt1@example.com", err) + } + if err := delivery.Body(context.Background(), textproto.Header{}, buffer.MemoryBuffer{Slice: []byte("foobar")}); err != nil { + t.Fatalf("unexpected Body err: %v", err) + } + if err := delivery.Commit(context.Background()); err != nil { + t.Fatalf("unexpected Commit err: %v", err) + } +} + +func TestMsgPipeline_PostmasterRcpt(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "postmaster": { + targets: []module.DeliveryTarget{&target}, + }, + "example.com": { + rejectErr: errors.New("go away"), + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("go away"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "disappointed-user@example.com", []string{"postmaster"}) + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 1, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "disappointed-user@example.com", []string{"postmaster"}) +} + +func TestMsgPipeline_PostmasterSrc(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "postmaster": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + "example.com": { + rejectErr: errors.New("go away"), + }, + }, + defaultSource: sourceBlock{ + rejectErr: errors.New("go away"), + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "postmaster", []string{"disappointed-user@example.com"}) + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 1, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "postmaster", []string{"disappointed-user@example.com"}) +} + +func TestMsgPipeline_CaseInsensetiveMatch_Src(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{ + "postmaster": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + "sender@example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + "example.com": { + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + defaultSource: sourceBlock{ + rejectErr: errors.New("go away"), + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "POSTMastER", []string{"disappointed-user@example.com"}) + testutils.DoTestDelivery(t, &d, "SenDeR@EXAMPLE.com", []string{"disappointed-user@example.com"}) + testutils.DoTestDelivery(t, &d, "sender@exAMPle.com", []string{"disappointed-user@example.com"}) + if len(target.Messages) != 3 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 3, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "POSTMastER", []string{"disappointed-user@example.com"}) + testutils.CheckTestMessage(t, &target, 1, "SenDeR@EXAMPLE.com", []string{"disappointed-user@example.com"}) + testutils.CheckTestMessage(t, &target, 2, "sender@exAMPle.com", []string{"disappointed-user@example.com"}) +} + +func TestMsgPipeline_CaseInsensetiveMatch_Rcpt(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "postmaster": { + targets: []module.DeliveryTarget{&target}, + }, + "sender@example.com": { + targets: []module.DeliveryTarget{&target}, + }, + "example.com": { + targets: []module.DeliveryTarget{&target}, + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("wtf"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"POSTMastER"}) + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"SenDeR@EXAMPLE.com"}) + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"sender@exAMPle.com"}) + if len(target.Messages) != 3 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 3, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"POSTMastER"}) + testutils.CheckTestMessage(t, &target, 1, "sender@example.com", []string{"SenDeR@EXAMPLE.com"}) + testutils.CheckTestMessage(t, &target, 2, "sender@example.com", []string{"sender@exAMPle.com"}) +} + +func TestMsgPipeline_UnicodeNFC_Rcpt(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "rcpt@é.example.com": { + targets: []module.DeliveryTarget{&target}, + }, + "é.example.com": { + targets: []module.DeliveryTarget{&target}, + }, + }, + defaultRcpt: &rcptBlock{ + rejectErr: errors.New("wtf"), + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"rcpt@E\u0301.EXAMPLE.com"}) + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"f@E\u0301.exAMPle.com"}) + if len(target.Messages) != 2 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 2, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"rcpt@E\u0301.EXAMPLE.com"}) + testutils.CheckTestMessage(t, &target, 1, "sender@example.com", []string{"f@E\u0301.exAMPle.com"}) +} + +func TestMsgPipeline_MalformedSource(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "postmaster": { + targets: []module.DeliveryTarget{&target}, + }, + "sender@example.com": { + targets: []module.DeliveryTarget{&target}, + }, + "example.com": { + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + // Simple checks for violations that can make msgpipeline misbehave. + for _, addr := range []string{"not_postmaster_but_no_at_sign", "@no_mailbox", "no_domain@"} { + _, err := d.Start(context.Background(), &module.MsgMetadata{ID: "testing"}, addr) + if err == nil { + t.Errorf("%s is accepted as valid address", addr) + } + } +} + +func TestMsgPipeline_TwoRcptToOneTarget(t *testing.T) { + target := testutils.Target{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{ + "example.com": { + targets: []module.DeliveryTarget{&target}, + }, + "example.org": { + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"recipient@example.com", "recipient@example.org"}) + + if len(target.Messages) != 1 { + t.Fatalf("wrong amount of messages received for target, want %d, got %d", 1, len(target.Messages)) + } + testutils.CheckTestMessage(t, &target, 0, "sender@example.com", []string{"recipient@example.com", "recipient@example.org"}) +} + +func TestMsgPipeline_multi_alias(t *testing.T) { + target1, target2 := testutils.Target{InstName: "target1"}, testutils.Target{InstName: "target2"} + mod := testutils.Modifier{ + RcptTo: map[string][]string{ + "recipient@example.com": []string{ + "recipient-1@example.org", + "recipient-2@example.net", + }, + }, + } + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + modifiers: modify.Group{ + Modifiers: []module.Modifier{mod}, + }, + perRcpt: map[string]*rcptBlock{ + "example.org": { + targets: []module.DeliveryTarget{&target1}, + }, + "example.net": { + targets: []module.DeliveryTarget{&target2}, + }, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "sender@example.com", []string{"recipient@example.com"}) + + if len(target1.Messages) != 1 { + t.Errorf("wrong amount of messages received for target1, want %d, got %d", 1, len(target1.Messages)) + } + testutils.CheckTestMessage(t, &target1, 0, "sender@example.com", []string{"recipient-1@example.org"}) + + if len(target2.Messages) != 1 { + t.Errorf("wrong amount of messages received for target1, want %d, got %d", 1, len(target2.Messages)) + } + testutils.CheckTestMessage(t, &target2, 0, "sender@example.com", []string{"recipient-2@example.net"}) +} diff --git a/internal/msgpipeline/objname.go b/internal/msgpipeline/objname.go new file mode 100644 index 0000000..9fc1b27 --- /dev/null +++ b/internal/msgpipeline/objname.go @@ -0,0 +1,46 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "fmt" + + "github.com/foxcpp/maddy/framework/module" +) + +// objectName returns a new that is usable to identify the used external +// component (module or some stub) in debug logs. +func objectName(x interface{}) string { + mod, ok := x.(module.Module) + if ok { + return mod.Name() + ":" + mod.InstanceName() + } + + _, pipeline := x.(*MsgPipeline) + if pipeline { + return "reroute" + } + + str, ok := x.(fmt.Stringer) + if ok { + return str.String() + } + + return fmt.Sprintf("%T", x) +} diff --git a/internal/msgpipeline/regress_test.go b/internal/msgpipeline/regress_test.go new file mode 100644 index 0000000..80e92ef --- /dev/null +++ b/internal/msgpipeline/regress_test.go @@ -0,0 +1,137 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package msgpipeline + +import ( + "testing" + + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestMsgPipeline_Issue161(t *testing.T) { + target := testutils.Target{} + check1, check2 := testutils.Check{}, testutils.Check{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&check1}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + checks: []module.Check{&check2}, + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "whatever@whatever", []string{"whatever@whatever"}) + + if check2.ConnCalls != 1 { + t.Errorf("CheckConnection called %d times", check2.ConnCalls) + } + if check2.SenderCalls != 1 { + t.Errorf("CheckSender called %d times", check2.SenderCalls) + } + if check2.RcptCalls != 1 { + t.Errorf("CheckRcpt called %d times", check2.RcptCalls) + } + if check2.BodyCalls != 1 { + t.Errorf("CheckBody called %d times", check2.BodyCalls) + } + + if check1.UnclosedStates != 0 || check2.UnclosedStates != 0 { + t.Fatalf("checks state objects leak or double-closed, alive counters: %v, %v", check1.UnclosedStates, check2.UnclosedStates) + } +} + +func TestMsgPipeline_Issue161_2(t *testing.T) { + target := testutils.Target{} + check1, check2 := testutils.Check{}, testutils.Check{InstName: "check2"} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&check1}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + checks: []module.Check{&check1}, + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + checks: []module.Check{&check2}, + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "whatever@whatever", []string{"whatever@whatever"}) + + if check2.ConnCalls != 1 { + t.Errorf("CheckConnection called %d times", check2.ConnCalls) + } + if check2.SenderCalls != 1 { + t.Errorf("CheckSender called %d times", check2.SenderCalls) + } + if check2.RcptCalls != 1 { + t.Errorf("CheckRcpt called %d times", check2.RcptCalls) + } + + if check1.UnclosedStates != 0 || check2.UnclosedStates != 0 { + t.Fatalf("checks state objects leak or double-closed, alive counters: %v, %v", check1.UnclosedStates, check2.UnclosedStates) + } +} + +func TestMsgPipeline_Issue161_3(t *testing.T) { + target := testutils.Target{} + check1, check2 := testutils.Check{}, testutils.Check{} + d := MsgPipeline{ + msgpipelineCfg: msgpipelineCfg{ + globalChecks: []module.Check{&check1, &check2}, + perSource: map[string]sourceBlock{}, + defaultSource: sourceBlock{ + perRcpt: map[string]*rcptBlock{}, + defaultRcpt: &rcptBlock{ + targets: []module.DeliveryTarget{&target}, + }, + }, + }, + Log: testutils.Logger(t, "msgpipeline"), + } + + testutils.DoTestDelivery(t, &d, "whatever@whatever", []string{"whatever@whatever"}) + + if check2.ConnCalls != 1 { + t.Errorf("CheckConnection called %d times", check2.ConnCalls) + } + if check2.SenderCalls != 1 { + t.Errorf("CheckSender called %d times", check2.SenderCalls) + } + if check2.RcptCalls != 1 { + t.Errorf("CheckRcpt called %d times", check2.RcptCalls) + } + if check2.BodyCalls != 1 { + t.Errorf("CheckBody called %d times", check2.BodyCalls) + } + + if check1.UnclosedStates != 0 || check2.UnclosedStates != 0 { + t.Fatalf("checks state objects leak or double-closed, alive counters: %v, %v", check1.UnclosedStates, check2.UnclosedStates) + } +} diff --git a/internal/proxy_protocol/proxy_protocol.go b/internal/proxy_protocol/proxy_protocol.go new file mode 100644 index 0000000..1a3a787 --- /dev/null +++ b/internal/proxy_protocol/proxy_protocol.go @@ -0,0 +1,86 @@ +package proxy_protocol + +import ( + "crypto/tls" + "net" + "strings" + + "github.com/c0va23/go-proxyprotocol" + "github.com/foxcpp/maddy/framework/config" + tls2 "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/log" +) + +type ProxyProtocol struct { + trust []net.IPNet + tlsConfig *tls.Config +} + +func ProxyProtocolDirective(_ *config.Map, node config.Node) (interface{}, error) { + p := ProxyProtocol{} + + childM := config.NewMap(nil, node) + var trustList []string + + childM.StringList("trust", false, false, nil, &trustList) + childM.Custom("tls", true, false, nil, tls2.TLSDirective, &p.tlsConfig) + + if _, err := childM.Process(); err != nil { + return nil, err + } + + if len(node.Args) > 0 { + if trustList == nil { + trustList = make([]string, 0) + } + trustList = append(trustList, node.Args...) + } + + for _, trust := range trustList { + if !strings.Contains(trust, "/") { + trust += "/32" + } + _, ipNet, err := net.ParseCIDR(trust) + if err != nil { + return nil, err + } + p.trust = append(p.trust, *ipNet) + } + + return &p, nil +} + +func NewListener(inner net.Listener, p *ProxyProtocol, logger log.Logger) net.Listener { + var listener net.Listener + + sourceChecker := func(upstream net.Addr) (bool, error) { + if tcpAddr, ok := upstream.(*net.TCPAddr); ok { + if len(p.trust) == 0 { + return true, nil + } + for _, trusted := range p.trust { + if trusted.Contains(tcpAddr.IP) { + return true, nil + } + } + } else if _, ok := upstream.(*net.UnixAddr); ok { + // UNIX local socket connection, always trusted + return true, nil + } + + logger.Printf("proxy_protocol: connection from untrusted source %s", upstream) + return false, nil + } + + listener = proxyprotocol.NewDefaultListener(inner). + WithLogger(proxyprotocol.LoggerFunc(func(format string, v ...interface{}) { + logger.Debugf("proxy_protocol: "+format, v...) + })). + WithSourceChecker(sourceChecker) + + if p.tlsConfig != nil { + listener = tls.NewListener(listener, p.tlsConfig) + } + + return listener +} diff --git a/internal/smtpconn/pool/pool.go b/internal/smtpconn/pool/pool.go new file mode 100644 index 0000000..4b700ee --- /dev/null +++ b/internal/smtpconn/pool/pool.go @@ -0,0 +1,211 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package pool + +import ( + "context" + "sync" + "time" +) + +type Conn interface { + Usable() bool + LastUseAt() time.Time + Close() error +} + +type Config struct { + New func(ctx context.Context, key string) (Conn, error) + MaxKeys int + MaxConnsPerKey int + MaxConnLifetimeSec int64 + StaleKeyLifetimeSec int64 +} + +type slot struct { + c chan Conn + // To keep slot size smaller it is just a unix timestamp. + lastUse int64 +} + +type P struct { + cfg Config + keys map[string]slot + keysLock sync.Mutex + + cleanupStop chan struct{} +} + +func New(cfg Config) *P { + if cfg.New == nil { + cfg.New = func(context.Context, string) (Conn, error) { + return nil, nil + } + } + + p := &P{ + cfg: cfg, + keys: make(map[string]slot, cfg.MaxKeys), + cleanupStop: make(chan struct{}), + } + + go p.cleanUpTick(p.cleanupStop) + + return p +} + +func (p *P) cleanUpTick(stop chan struct{}) { + ctx := context.Background() + tick := time.NewTicker(time.Minute) + defer tick.Stop() + + for { + select { + case <-tick.C: + p.CleanUp(ctx) + case <-stop: + return + } + } +} + +func (p *P) CleanUp(ctx context.Context) { + p.keysLock.Lock() + defer p.keysLock.Unlock() + + for k, v := range p.keys { + if v.lastUse+p.cfg.StaleKeyLifetimeSec > time.Now().Unix() { + continue + } + + close(v.c) + for conn := range v.c { + go conn.Close() + } + delete(p.keys, k) + } +} + +func (p *P) Get(ctx context.Context, key string) (Conn, error) { + p.keysLock.Lock() + + bucket, ok := p.keys[key] + if !ok { + p.keysLock.Unlock() + return p.cfg.New(ctx, key) + } + + if time.Now().Unix()-bucket.lastUse > p.cfg.MaxConnLifetimeSec { + // Drop bucket. + delete(p.keys, key) + close(bucket.c) + + // Close might take some time, unlock early. + p.keysLock.Unlock() + + for conn := range bucket.c { + conn.Close() + } + + return p.cfg.New(ctx, key) + } + + p.keysLock.Unlock() + + for { + var conn Conn + select { + case conn, ok = <-bucket.c: + if !ok { + return p.cfg.New(ctx, key) + } + default: + return p.cfg.New(ctx, key) + } + + if !conn.Usable() { + // Close might take some time, run in parallel. + go conn.Close() + continue + } + if conn.LastUseAt().Add(time.Duration(p.cfg.MaxConnLifetimeSec) * time.Second).Before(time.Now()) { + go conn.Close() + continue + } + + return conn, nil + } +} + +func (p *P) Return(key string, c Conn) { + p.keysLock.Lock() + defer p.keysLock.Unlock() + + if p.keys == nil { + return + } + + bucket, ok := p.keys[key] + if !ok { + // Garbage-collect stale buckets. + if len(p.keys) == p.cfg.MaxKeys { + for k, v := range p.keys { + if v.lastUse+p.cfg.StaleKeyLifetimeSec > time.Now().Unix() { + continue + } + delete(p.keys, k) + close(v.c) + + for conn := range v.c { + conn.Close() + } + } + } + + bucket = slot{ + c: make(chan Conn, p.cfg.MaxConnsPerKey), + lastUse: time.Now().Unix(), + } + p.keys[key] = bucket + } + + select { + case bucket.c <- c: + bucket.lastUse = time.Now().Unix() + default: + // Let it go, let it go... + go c.Close() + } +} + +func (p *P) Close() { + p.cleanupStop <- struct{}{} + + p.keysLock.Lock() + defer p.keysLock.Unlock() + + for k, v := range p.keys { + close(v.c) + for conn := range v.c { + conn.Close() + } + delete(p.keys, k) + } + p.keys = nil +} diff --git a/internal/smtpconn/smtpconn.go b/internal/smtpconn/smtpconn.go new file mode 100644 index 0000000..ec42974 --- /dev/null +++ b/internal/smtpconn/smtpconn.go @@ -0,0 +1,558 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package smtpconn contains the code shared between target.smtp and +// remote modules. +// +// It implements the wrapper over the SMTP connection (go-smtp.Client) object +// with the following features added: +// - Logging of certain errors (e.g. QUIT command errors) +// - Wrapping of returned errors using the exterrors package. +// - SMTPUTF8/IDNA support. +// - TLS support mode (don't use, attempt, require). +package smtpconn + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "runtime/trace" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" +) + +// The C object represents the SMTP connection and is a wrapper around +// go-smtp.Client with additional maddy-specific logic. +// +// Currently, the C object represents one session and cannot be reused. +type C struct { + // Dialer to use to estabilish new network connections. Set to net.Dialer + // DialContext by New. + Dialer func(ctx context.Context, network, addr string) (net.Conn, error) + + // Timeout for most session commands (EHLO, MAIL, RCPT, DATA, STARTTLS). + // Set to 5 mins by New. + CommandTimeout time.Duration + + // Timeout for the initial TCP connection establishment. + ConnectTimeout time.Duration + + // Timeout for the final dot. Set to 12 mins by New. + // (see go-smtp source for explanation of used defaults). + SubmissionTimeout time.Duration + + // Hostname to sent in the EHLO/HELO command. Set to + // 'localhost.localdomain' by New. Expected to be encoded in ACE form. + Hostname string + + // tls.Config to use. Can be nil if no special changes are required. + TLSConfig *tls.Config + + // Logger to use for debug log and certain errors. + Log log.Logger + + // Include the remote server address in SMTP status messages in the form + // "ADDRESS said: ..." + AddrInSMTPMsg bool + + conn net.Conn + serverName string + cl *smtp.Client + rcpts []string + lmtp bool +} + +// New creates the new instance of the C object, populating the required fields +// with resonable default values. +func New() *C { + return &C{ + Dialer: (&net.Dialer{}).DialContext, + ConnectTimeout: 5 * time.Minute, + CommandTimeout: 5 * time.Minute, + SubmissionTimeout: 12 * time.Minute, + TLSConfig: &tls.Config{}, + Hostname: "localhost.localdomain", + } +} + +func (c *C) wrapClientErr(err error, serverName string) error { + if err == nil { + return nil + } + + switch err := err.(type) { + case TLSError: + return err + case *exterrors.SMTPError: + return err + case *smtp.SMTPError: + msg := err.Message + if c.AddrInSMTPMsg { + msg = serverName + " said: " + err.Message + } + + if err.Code == 552 { + err.Code = 452 + err.EnhancedCode[0] = 4 + c.Log.Msg("SMTP code 552 rewritten to 452 per RFC 5321 Section 4.5.3.1.10") + } + + return &exterrors.SMTPError{ + Code: err.Code, + EnhancedCode: exterrors.EnhancedCode(err.EnhancedCode), + Message: msg, + Misc: map[string]interface{}{ + "remote_server": serverName, + }, + Err: err, + } + case *net.OpError: + if _, ok := err.Err.(*net.DNSError); ok { + reason, misc := exterrors.UnwrapDNSErr(err) + misc["remote_server"] = err.Addr + misc["io_op"] = err.Op + return &exterrors.SMTPError{ + Code: exterrors.SMTPCode(err, 450, 550), + EnhancedCode: exterrors.SMTPEnchCode(err, exterrors.EnhancedCode{0, 4, 4}), + Message: "DNS error", + Err: err, + Reason: reason, + Misc: misc, + } + } + return &exterrors.SMTPError{ + Code: 450, + EnhancedCode: exterrors.EnhancedCode{4, 4, 2}, + Message: "Network I/O error", + Err: err, + Misc: map[string]interface{}{ + "remote_addr": err.Addr, + "io_op": err.Op, + }, + } + default: + return exterrors.WithFields(err, map[string]interface{}{ + "remote_server": serverName, + }) + } +} + +// Connect actually estabilishes the network connection with the remote host, +// executes HELO/EHLO and optionally STARTTLS command. +func (c *C) Connect(ctx context.Context, endp config.Endpoint, starttls bool, tlsConfig *tls.Config) (didTLS bool, err error) { + didTLS, cl, conn, err := c.attemptConnect(ctx, false, endp, starttls, tlsConfig) + if err != nil { + return false, c.wrapClientErr(err, endp.Host) + } + + c.serverName = endp.Host + c.cl = cl + c.conn = conn + + c.Log.DebugMsg("connected", "remote_server", c.serverName, + "local_addr", c.LocalAddr(), "remote_addr", c.RemoteAddr()) + + return didTLS, nil +} + +// ConnectLMTP estabilishes the network connection with the remote host and +// sends LHLO command, negotiating LMTP use. +func (c *C) ConnectLMTP(ctx context.Context, endp config.Endpoint, starttls bool, tlsConfig *tls.Config) (didTLS bool, err error) { + didTLS, cl, conn, err := c.attemptConnect(ctx, true, endp, starttls, tlsConfig) + if err != nil { + return false, c.wrapClientErr(err, endp.Host) + } + + c.serverName = endp.Host + c.cl = cl + c.conn = conn + + c.Log.DebugMsg("connected", "remote_server", c.serverName, + "local_addr", c.LocalAddr(), "remote_addr", c.RemoteAddr()) + + return didTLS, nil +} + +// TLSError is returned by Connect to indicate the error during STARTTLS +// command execution. +// +// If the endpoint uses Implicit TLS, TLS errors are threated as connection +// errors and thus are not returned as TLSError. +type TLSError struct { + Err error +} + +func (err TLSError) Error() string { + return "smtpconn: " + err.Err.Error() +} + +func (err TLSError) Unwrap() error { + return err.Err +} + +func (c *C) LocalAddr() net.Addr { + if c.conn == nil { + return nil + } + return c.conn.LocalAddr() +} + +func (c *C) RemoteAddr() net.Addr { + if c.conn == nil { + return nil + } + return c.conn.RemoteAddr() +} + +func (c *C) attemptConnect(ctx context.Context, lmtp bool, endp config.Endpoint, starttls bool, tlsConfig *tls.Config) (didTLS bool, cl *smtp.Client, conn net.Conn, err error) { + dialCtx, cancel := context.WithTimeout(ctx, c.ConnectTimeout) + conn, err = c.Dialer(dialCtx, endp.Network(), endp.Address()) + cancel() + if err != nil { + return false, nil, nil, err + } + + if endp.IsTLS() { + cfg := tlsConfig.Clone() + cfg.ServerName = endp.Host + conn = tls.Client(conn, cfg) + } + + c.lmtp = lmtp + // This uses initial greeting timeout of 5 minutes (hardcoded). + if lmtp { + cl = smtp.NewClientLMTP(conn) + } else { + cl = smtp.NewClient(conn) + } + + cl.CommandTimeout = c.CommandTimeout + cl.SubmissionTimeout = c.SubmissionTimeout + + // i18n: hostname is already expected to be in A-labels form. + if err := cl.Hello(c.Hostname); err != nil { + cl.Close() + return false, nil, nil, err + } + + if !starttls { + return false, cl, conn, nil + } + + if ok, _ := cl.Extension("STARTTLS"); !ok { + if err := cl.Quit(); err != nil { + cl.Close() + } + return false, nil, nil, fmt.Errorf("TLS required but unsupported by downstream") + } + + cfg := tlsConfig.Clone() + cfg.ServerName = endp.Host + if err := cl.StartTLS(cfg); err != nil { + // After the handshake failure, the connection may be in a bad state. + // We attempt to send the proper QUIT command though, in case the error happened + // *after* the handshake (e.g. PKI verification fail), we don't log the error in + // this case though. + if err := cl.Quit(); err != nil { + cl.Close() + } + + return false, nil, nil, TLSError{err} + } + + // Re-do HELO using our hostname instead of localhost. + if err := cl.Hello(c.Hostname); err != nil { + cl.Close() + + var tlsErr *tls.CertificateVerificationError + if errors.As(err, &tlsErr) { + return false, nil, nil, TLSError{Err: tlsErr} + } + + return false, nil, nil, err + } + + return true, cl, conn, nil +} + +// Mail sends the MAIL FROM command to the remote server. +// +// SIZE and REQUIRETLS options are forwarded to the remote server as-is. +// SMTPUTF8 is forwarded if supported by the remote server, if it is not +// supported - attempt will be done to convert addresses to the ASCII form, if +// this is not possible, the corresponding method (Mail or Rcpt) will fail. +func (c *C) Mail(ctx context.Context, from string, opts smtp.MailOptions) error { + defer trace.StartRegion(ctx, "smtpconn/MAIL FROM").End() + + outOpts := smtp.MailOptions{ + // Future extensions may add additional fields that should not be + // copied blindly. So we copy only fields we know should be handled + // this way. + + Size: opts.Size, + RequireTLS: opts.RequireTLS, + } + + // INTERNATIONALIZATION: Use SMTPUTF8 is possible, attempt to convert addresses otherwise. + + // There is no way we can accept a message with non-ASCII addresses without SMTPUTF8 + // this is enforced by endpoint/smtp. + if opts.UTF8 { + if ok, _ := c.cl.Extension("SMTPUTF8"); ok { + outOpts.UTF8 = true + } else { + var err error + from, err = address.ToASCII(from) + if err != nil { + return &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 6, 7}, + Message: "SMTPUTF8 is unsupported, cannot convert sender address", + Misc: map[string]interface{}{ + "remote_server": c.serverName, + }, + Err: err, + } + } + } + } + + if err := c.cl.Mail(from, &outOpts); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + return nil +} + +// Rcpts returns the list of recipients that were accepted by the remote server. +func (c *C) Rcpts() []string { + return c.rcpts +} + +func (c *C) ServerName() string { + return c.serverName +} + +func (c *C) Client() *smtp.Client { + return c.cl +} + +func (c *C) IsLMTP() bool { + return c.lmtp +} + +// Rcpt sends the RCPT TO command to the remote server. +// +// If the address is non-ASCII and cannot be converted to ASCII and the remote +// server does not support SMTPUTF8, error will be returned. +func (c *C) Rcpt(ctx context.Context, to string, opts smtp.RcptOptions) error { + defer trace.StartRegion(ctx, "smtpconn/RCPT TO").End() + + outOpts := &smtp.RcptOptions{ + // TODO: DSN support + } + + // If necessary, the extension flag is enabled in Start. + if ok, _ := c.cl.Extension("SMTPUTF8"); !address.IsASCII(to) && !ok { + var err error + to, err = address.ToASCII(to) + if err != nil { + return &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 6, 7}, + Message: "SMTPUTF8 is unsupported, cannot convert recipient address", + Misc: map[string]interface{}{ + "remote_server": c.serverName, + }, + Err: err, + } + } + } + + if err := c.cl.Rcpt(to, outOpts); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + c.rcpts = append(c.rcpts, to) + + return nil +} + +type lmtpError map[string]*smtp.SMTPError + +func (l lmtpError) SetStatus(rcptTo string, err *smtp.SMTPError) { + l[rcptTo] = err +} + +func (l lmtpError) singleError() *smtp.SMTPError { + nonNils := 0 + for _, e := range l { + if e != nil { + nonNils++ + } + } + if nonNils == 1 { + for _, err := range l { + if err != nil { + return err + } + } + } + return nil +} + +func (l lmtpError) Unwrap() error { + if err := l.singleError(); err != nil { + return err + } + return nil +} + +func (l lmtpError) Error() string { + if err := l.singleError(); err != nil { + return err.Error() + } + return fmt.Sprintf("multiple errors reported by LMTP downstream: %v", map[string]*smtp.SMTPError(l)) +} + +func (c *C) smtpToLMTPData(ctx context.Context, hdr textproto.Header, body io.Reader) error { + statusCb := lmtpError{} + if err := c.LMTPData(ctx, hdr, body, statusCb.SetStatus); err != nil { + return err + } + hasAnyFailures := false + for _, err := range statusCb { + if err != nil { + hasAnyFailures = true + } + } + if hasAnyFailures { + return statusCb + } + return nil +} + +// Data sends the DATA command to the remote server and then sends the message header +// and body. +// +// If the Data command fails, the connection may be in a unclean state (e.g. in +// the middle of message data stream). It is not safe to continue using it. +func (c *C) Data(ctx context.Context, hdr textproto.Header, body io.Reader) error { + defer trace.StartRegion(ctx, "smtpconn/DATA").End() + + if c.IsLMTP() { + return c.smtpToLMTPData(ctx, hdr, body) + } + + wc, err := c.cl.Data() + if err != nil { + return c.wrapClientErr(err, c.serverName) + } + + if err := textproto.WriteHeader(wc, hdr); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + if _, err := io.Copy(wc, body); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + if err := wc.Close(); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + return nil +} + +func (c *C) LMTPData(ctx context.Context, hdr textproto.Header, body io.Reader, statusCb func(string, *smtp.SMTPError)) error { + defer trace.StartRegion(ctx, "smtpconn/LMTPDATA").End() + + wc, err := c.cl.LMTPData(statusCb) + if err != nil { + return c.wrapClientErr(err, c.serverName) + } + + if err := textproto.WriteHeader(wc, hdr); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + if _, err := io.Copy(wc, body); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + if err := wc.Close(); err != nil { + return c.wrapClientErr(err, c.serverName) + } + + return nil +} + +func (c *C) Noop() error { + if c.cl == nil { + return errors.New("smtpconn: not connected") + } + + return c.cl.Noop() +} + +// Close sends the QUIT command, if it fails - it directly closes the +// connection. +func (c *C) Close() error { + c.cl.CommandTimeout = 5 * time.Second + + if err := c.cl.Quit(); err != nil { + var smtpErr *smtp.SMTPError + var netErr *net.OpError + if errors.As(err, &smtpErr) && smtpErr.Code == 421 { + // 421 "Service not available" is typically sent + // when idle timeout happens. + c.Log.DebugMsg("QUIT error", "reason", c.wrapClientErr(err, c.serverName)) + } else if errors.As(err, &netErr) && + (netErr.Timeout() || netErr.Err.Error() == "write: broken pipe" || netErr.Err.Error() == "read: connection reset") { + // The case for silently closed connections. + c.Log.DebugMsg("QUIT error", "reason", c.wrapClientErr(err, c.serverName)) + } else { + c.Log.Error("QUIT error", c.wrapClientErr(err, c.serverName)) + } + + return c.cl.Close() + } + + c.cl = nil + c.serverName = "" + + return nil +} + +// DirectClose closes the underlying connection without sending the QUIT +// command. +func (c *C) DirectClose() error { + c.cl.Close() + c.cl = nil + c.serverName = "" + return nil +} diff --git a/internal/smtpconn/smtpconn_test.go b/internal/smtpconn/smtpconn_test.go new file mode 100644 index 0000000..b8fd647 --- /dev/null +++ b/internal/smtpconn/smtpconn_test.go @@ -0,0 +1,41 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtpconn + +import ( + "flag" + "math/rand" + "os" + "strconv" + "testing" +) + +var testPort string + +func TestMain(m *testing.M) { + remoteSmtpPort := flag.String("test.smtpport", "random", "(maddy) SMTP port to use for connections in tests") + flag.Parse() + + if *remoteSmtpPort == "random" { + *remoteSmtpPort = strconv.Itoa(rand.Intn(65536-10000) + 10000) + } + + testPort = *remoteSmtpPort + os.Exit(m.Run()) +} diff --git a/internal/smtpconn/smtputf8_test.go b/internal/smtpconn/smtputf8_test.go new file mode 100644 index 0000000..dc580d9 --- /dev/null +++ b/internal/smtpconn/smtputf8_test.go @@ -0,0 +1,165 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtpconn + +import ( + "context" + "strings" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/internal/testutils" +) + +func doTestDelivery(t *testing.T, conn *C, from string, to []string, opts smtp.MailOptions) error { + t.Helper() + + if err := conn.Mail(context.Background(), from, opts); err != nil { + return err + } + for _, rcpt := range to { + if err := conn.Rcpt(context.Background(), rcpt, smtp.RcptOptions{}); err != nil { + return err + } + } + + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + return conn.Data(context.Background(), hdr, strings.NewReader("foobar\n")) +} + +func TestSMTPUTF8(t *testing.T) { + type test struct { + clientSender string + clientRcpt string + + serverUTF8 bool + serverSender string + serverRcpt string + + expectUTF8 bool + expectErr *exterrors.SMTPError + } + check := func(case_ test) { + t.Helper() + + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + srv.EnableSMTPUTF8 = case_.serverUTF8 + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + c := New() + c.Log = testutils.Logger(t, "target.smtp") + if _, err := c.Connect(context.Background(), config.Endpoint{ + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, false, nil); err != nil { + t.Fatal(err) + } + defer c.Close() + + err := doTestDelivery(t, c, case_.clientSender, []string{case_.clientRcpt}, + smtp.MailOptions{UTF8: true}) + if err != nil { + if case_.expectErr == nil { + t.Error("Unexpected failure") + } else { + testutils.CheckSMTPErr(t, err, case_.expectErr.Code, case_.expectErr.EnhancedCode, case_.expectErr.Message) + } + return + } else if case_.expectErr != nil { + t.Error("Unexpected success") + } + + be.CheckMsg(t, 0, case_.serverSender, []string{case_.serverRcpt}) + if be.Messages[0].Opts.UTF8 != case_.expectUTF8 { + t.Errorf("expectUTF8 = %v, SMTPUTF8 = %v", case_.expectErr, be.Messages[0].Opts.UTF8) + } + } + + check(test{ + clientSender: "test@тест.example.org", + clientRcpt: "test@example.invalid", + serverSender: "test@xn--e1aybc.example.org", + serverRcpt: "test@example.invalid", + }) + check(test{ + clientSender: "test@example.org", + clientRcpt: "test@тест.example.invalid", + serverSender: "test@example.org", + serverRcpt: "test@xn--e1aybc.example.invalid", + }) + check(test{ + clientSender: "тест@example.org", + clientRcpt: "test@example.invalid", + serverUTF8: false, + expectErr: &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 6, 7}, + Message: "SMTPUTF8 is unsupported, cannot convert sender address", + }, + }) + check(test{ + clientSender: "test@example.org", + clientRcpt: "тест@example.invalid", + serverUTF8: false, + expectErr: &exterrors.SMTPError{ + Code: 553, + EnhancedCode: exterrors.EnhancedCode{5, 6, 7}, + Message: "SMTPUTF8 is unsupported, cannot convert recipient address", + }, + }) + check(test{ + clientSender: "test@тест.org", + clientRcpt: "test@example.invalid", + serverSender: "test@тест.org", + serverRcpt: "test@example.invalid", + serverUTF8: true, + expectUTF8: true, + }) + check(test{ + clientSender: "test@example.org", + clientRcpt: "test@тест.example.invalid", + serverSender: "test@example.org", + serverRcpt: "test@тест.example.invalid", + serverUTF8: true, + expectUTF8: true, + }) + check(test{ + clientSender: "тест@example.org", + clientRcpt: "test@example.invalid", + serverSender: "тест@example.org", + serverRcpt: "test@example.invalid", + serverUTF8: true, + expectUTF8: true, + }) + check(test{ + clientSender: "test@example.org", + clientRcpt: "тест@example.invalid", + serverSender: "test@example.org", + serverRcpt: "тест@example.invalid", + serverUTF8: true, + expectUTF8: true, + }) +} diff --git a/internal/storage/blob/fs/fs.go b/internal/storage/blob/fs/fs.go new file mode 100644 index 0000000..e8c9b38 --- /dev/null +++ b/internal/storage/blob/fs/fs.go @@ -0,0 +1,95 @@ +package fs + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +// FSStore struct represents directory on FS used to store blobs. +type FSStore struct { + instName string + root string +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + switch len(inlineArgs) { + case 0: + return &FSStore{instName: instName}, nil + case 1: + return &FSStore{instName: instName, root: inlineArgs[0]}, nil + default: + return nil, fmt.Errorf("storage.blob.fs: 1 or 0 arguments expected") + } +} + +func (s FSStore) Name() string { + return "storage.blob.fs" +} + +func (s FSStore) InstanceName() string { + return s.instName +} + +func (s *FSStore) Init(cfg *config.Map) error { + cfg.String("root", false, false, s.root, &s.root) + if _, err := cfg.Process(); err != nil { + return err + } + + if s.root == "" { + return config.NodeErr(cfg.Block, "storage.blob.fs: directory not set") + } + + if err := os.MkdirAll(s.root, os.ModeDir|os.ModePerm); err != nil { + return err + } + + return nil +} + +func (s *FSStore) Open(_ context.Context, key string) (io.ReadCloser, error) { + f, err := os.Open(filepath.Join(s.root, key)) + if err != nil { + if os.IsNotExist(err) { + return nil, module.ErrNoSuchBlob + } + return nil, err + } + return f, nil +} + +func (s *FSStore) Create(_ context.Context, key string, blobSize int64) (module.Blob, error) { + f, err := os.Create(filepath.Join(s.root, key)) + if err != nil { + return nil, err + } + if blobSize >= 0 { + if err := f.Truncate(blobSize); err != nil { + return nil, err + } + } + return f, nil +} + +func (s *FSStore) Delete(_ context.Context, keys []string) error { + for _, key := range keys { + if err := os.Remove(filepath.Join(s.root, key)); err != nil { + if os.IsNotExist(err) { + continue + } + return err + } + } + return nil +} + +func init() { + var _ module.BlobStore = &FSStore{} + module.Register(FSStore{}.Name(), New) +} diff --git a/internal/storage/blob/fs/fs_test.go b/internal/storage/blob/fs/fs_test.go new file mode 100644 index 0000000..2c8f766 --- /dev/null +++ b/internal/storage/blob/fs/fs_test.go @@ -0,0 +1,19 @@ +package fs + +import ( + "os" + "testing" + + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/storage/blob" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestFS(t *testing.T) { + blob.TestStore(t, func() module.BlobStore { + dir := testutils.Dir(t) + return &FSStore{instName: "test", root: dir} + }, func(store module.BlobStore) { + os.RemoveAll(store.(*FSStore).root) + }) +} diff --git a/internal/storage/blob/s3/s3.go b/internal/storage/blob/s3/s3.go new file mode 100644 index 0000000..af01d88 --- /dev/null +++ b/internal/storage/blob/s3/s3.go @@ -0,0 +1,196 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "net/http" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +const modName = "storage.blob.s3" + +const ( + credsTypeFileMinio = "file_minio" + credsTypeFileAWS = "file_aws" + credsTypeAccessKey = "access_key" + credsTypeIAM = "iam" + credsTypeDefault = credsTypeAccessKey +) + +type Store struct { + instName string + log log.Logger + + endpoint string + cl *minio.Client + + bucketName string + objectPrefix string +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, fmt.Errorf("%s: expected 0 arguments", modName) + } + + return &Store{ + instName: instName, + log: log.Logger{Name: modName}, + }, nil +} + +func (s *Store) Init(cfg *config.Map) error { + var ( + secure bool + accessKeyID string + secretAccessKey string + credsType string + location string + ) + cfg.String("endpoint", false, true, "", &s.endpoint) + cfg.Bool("secure", false, true, &secure) + cfg.String("access_key", false, true, "", &accessKeyID) + cfg.String("secret_key", false, true, "", &secretAccessKey) + cfg.String("bucket", false, true, "", &s.bucketName) + cfg.String("region", false, false, "", &location) + cfg.String("object_prefix", false, false, "", &s.objectPrefix) + cfg.String("creds", false, false, credsTypeDefault, &credsType) + + if _, err := cfg.Process(); err != nil { + return err + } + if s.endpoint == "" { + return fmt.Errorf("%s: endpoint not set", modName) + } + + var creds *credentials.Credentials + + switch credsType { + case credsTypeFileMinio: + creds = credentials.NewFileMinioClient("", "") + case credsTypeFileAWS: + creds = credentials.NewFileAWSCredentials("", "") + case credsTypeIAM: + creds = credentials.NewIAM("") + case credsTypeAccessKey: + creds = credentials.NewStaticV4(accessKeyID, secretAccessKey, "") + default: + creds = credentials.NewStaticV4(accessKeyID, secretAccessKey, "") + } + + cl, err := minio.New(s.endpoint, &minio.Options{ + Creds: creds, + Secure: secure, + Region: location, + }) + if err != nil { + return fmt.Errorf("%s: %w", modName, err) + } + + s.cl = cl + return nil +} + +func (s *Store) Name() string { + return modName +} + +func (s *Store) InstanceName() string { + return s.instName +} + +type s3blob struct { + pw *io.PipeWriter + didSync bool + errCh chan error +} + +func (b *s3blob) Sync() error { + // We do this in Sync instead of Close because + // backend may not actually check the error of Close. + // The problematic restriction is that Sync can now be called + // only once. + if b.didSync { + panic("storage.blob.s3: Sync called twice for a blob object") + } + + b.pw.Close() + b.didSync = true + return <-b.errCh +} + +func (b *s3blob) Write(p []byte) (n int, err error) { + return b.pw.Write(p) +} + +func (b *s3blob) Close() error { + if !b.didSync { + if err := b.pw.CloseWithError(fmt.Errorf("storage.blob.s3: blob closed without Sync")); err != nil { + panic(err) + } + } + return nil +} + +func (s *Store) Create(ctx context.Context, key string, blobSize int64) (module.Blob, error) { + pr, pw := io.Pipe() + errCh := make(chan error, 1) + + go func() { + partSize := uint64(0) + if blobSize == module.UnknownBlobSize { + // Without this, minio-go will allocate 500 MiB buffer which + // is a little too much. + // https://github.com/minio/minio-go/issues/1478 + partSize = 1 * 1024 * 1024 /* 1 MiB */ + } + _, err := s.cl.PutObject(ctx, s.bucketName, s.objectPrefix+key, pr, blobSize, minio.PutObjectOptions{ + PartSize: partSize, + }) + if err != nil { + if err := pr.CloseWithError(fmt.Errorf("s3 PutObject: %w", err)); err != nil { + panic(err) + } + } + errCh <- err + }() + + return &s3blob{ + pw: pw, + errCh: errCh, + }, nil +} + +func (s *Store) Open(ctx context.Context, key string) (io.ReadCloser, error) { + obj, err := s.cl.GetObject(ctx, s.bucketName, s.objectPrefix+key, minio.GetObjectOptions{}) + if err != nil { + resp := minio.ToErrorResponse(err) + if resp.StatusCode == http.StatusNotFound { + return nil, module.ErrNoSuchBlob + } + return nil, err + } + return obj, nil +} + +func (s *Store) Delete(ctx context.Context, keys []string) error { + var lastErr error + for _, k := range keys { + lastErr = s.cl.RemoveObject(ctx, s.bucketName, s.objectPrefix+k, minio.RemoveObjectOptions{}) + if lastErr != nil { + s.log.Error("failed to delete object", lastErr, s.objectPrefix+k) + } + } + return lastErr +} + +func init() { + var _ module.BlobStore = &Store{} + module.Register(modName, New) +} diff --git a/internal/storage/blob/s3/s3_test.go b/internal/storage/blob/s3/s3_test.go new file mode 100644 index 0000000..98dd228 --- /dev/null +++ b/internal/storage/blob/s3/s3_test.go @@ -0,0 +1,71 @@ +package s3 + +import ( + "net/http/httptest" + "testing" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/storage/blob" + "github.com/johannesboyne/gofakes3" + "github.com/johannesboyne/gofakes3/backend/s3mem" +) + +func TestFS(t *testing.T) { + var ( + backend gofakes3.Backend + faker *gofakes3.GoFakeS3 + ts *httptest.Server + ) + + blob.TestStore(t, func() module.BlobStore { + backend = s3mem.New() + faker = gofakes3.New(backend) + ts = httptest.NewServer(faker.Server()) + + if err := backend.CreateBucket("maddy-test"); err != nil { + panic(err) + } + + st := &Store{instName: "test"} + err := st.Init(config.NewMap(map[string]interface{}{}, config.Node{ + Children: []config.Node{ + { + Name: "endpoint", + Args: []string{ts.Listener.Addr().String()}, + }, + { + Name: "secure", + Args: []string{"false"}, + }, + { + Name: "access_key", + Args: []string{"access-key"}, + }, + { + Name: "secret_key", + Args: []string{"secret-key"}, + }, + { + Name: "bucket", + Args: []string{"maddy-test"}, + }, + }, + })) + if err != nil { + panic(err) + } + + return st + }, func(store module.BlobStore) { + ts.Close() + + backend = s3mem.New() + faker = gofakes3.New(backend) + ts = httptest.NewServer(faker.Server()) + }) + + if ts != nil { + ts.Close() + } +} diff --git a/internal/storage/blob/test_blob.go b/internal/storage/blob/test_blob.go new file mode 100644 index 0000000..9672efe --- /dev/null +++ b/internal/storage/blob/test_blob.go @@ -0,0 +1,62 @@ +//go:build cgo && !no_sqlite3 +// +build cgo,!no_sqlite3 + +package blob + +import ( + "math/rand" + "testing" + + backendtests "github.com/foxcpp/go-imap-backend-tests" + imapsql "github.com/foxcpp/go-imap-sql" + "github.com/foxcpp/maddy/framework/module" + imapsql2 "github.com/foxcpp/maddy/internal/storage/imapsql" + "github.com/foxcpp/maddy/internal/testutils" +) + +type testBack struct { + backendtests.Backend + ExtStore module.BlobStore +} + +func TestStore(t *testing.T, newStore func() module.BlobStore, cleanStore func(module.BlobStore)) { + // We use go-imap-sql backend and run a subset of + // go-imap-backend-tests related to loading and saving messages. + // + // In the future we should probably switch to using a memory + // backend for this. + + backendtests.Whitelist = []string{ + t.Name() + "/Mailbox_CreateMessage", + t.Name() + "/Mailbox_ListMessages_Body", + t.Name() + "/Mailbox_CopyMessages", + t.Name() + "/Mailbox_Expunge", + t.Name() + "/Mailbox_MoveMessages", + } + + initBackend := func() backendtests.Backend { + randSrc := rand.NewSource(0) + prng := rand.New(randSrc) + store := newStore() + + b, err := imapsql.New("sqlite3", ":memory:", + imapsql2.ExtBlobStore{Base: store}, imapsql.Opts{ + PRNG: prng, + Log: testutils.Logger(t, "imapsql"), + }, + ) + if err != nil { + panic(err) + } + return testBack{Backend: b, ExtStore: store} + } + cleanBackend := func(bi backendtests.Backend) { + b := bi.(testBack) + if err := b.Backend.(*imapsql.Backend).Close(); err != nil { + panic(err) + } + cleanStore(b.ExtStore) + } + + backendtests.RunTests(t, initBackend, cleanBackend) +} diff --git a/internal/storage/blob/test_blob_nosqlite.go b/internal/storage/blob/test_blob_nosqlite.go new file mode 100644 index 0000000..601f467 --- /dev/null +++ b/internal/storage/blob/test_blob_nosqlite.go @@ -0,0 +1,14 @@ +//go:build !cgo || no_sqlite3 +// +build !cgo no_sqlite3 + +package blob + +import ( + "testing" + + "github.com/foxcpp/maddy/framework/module" +) + +func TestStore(t *testing.T, newStore func() module.BlobStore, cleanStore func(module.BlobStore)) { + t.Skip("storage.blob tests require CGo and sqlite3") +} diff --git a/internal/storage/imapsql/bench_test.go b/internal/storage/imapsql/bench_test.go new file mode 100644 index 0000000..04cfac9 --- /dev/null +++ b/internal/storage/imapsql/bench_test.go @@ -0,0 +1,90 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imapsql + +import ( + "flag" + "strconv" + "testing" + "time" + + imapsql "github.com/foxcpp/go-imap-sql" + "github.com/foxcpp/maddy/internal/testutils" +) + +var ( + testDB string + testDSN string + testFsstore string +) + +func init() { + flag.StringVar(&testDB, "sql.testdb", "", "Database to use for storage/sql benchmarks") + flag.StringVar(&testDSN, "sql.testdsn", "", "DSN to use for storage/sql benchmarks") + flag.StringVar(&testFsstore, "sql.testfsstore", "", "fsstore location to use for storage/sql benchmarks") +} + +func createTestDB(tb testing.TB, compAlgo string) *Storage { + if testDB == "" || testDSN == "" || testFsstore == "" { + tb.Skip("-sql.testdb, -sql.testdsn and -sql.testfsstore should be specified to run this benchmark") + } + + db, err := imapsql.New(testDB, testDSN, &imapsql.FSStore{Root: testFsstore}, imapsql.Opts{ + CompressAlgo: compAlgo, + }) + if err != nil { + tb.Fatal(err) + } + return &Storage{ + Back: db, + } +} + +func BenchmarkStorage_Delivery(b *testing.B) { + randomKey := "rcpt-" + strconv.FormatInt(time.Now().UnixNano(), 10) + "@example.org" + + be := createTestDB(b, "") + if err := be.CreateIMAPAcct(randomKey); err != nil { + b.Fatal(err) + } + + testutils.BenchDelivery(b, be, "sender@example.org", []string{randomKey}) +} + +func BenchmarkStorage_DeliveryLZ4(b *testing.B) { + randomKey := "rcpt-" + strconv.FormatInt(time.Now().UnixNano(), 10) + "@example.org" + + be := createTestDB(b, "lz4") + if err := be.CreateIMAPAcct(randomKey); err != nil { + b.Fatal(err) + } + + testutils.BenchDelivery(b, be, "sender@example.org", []string{randomKey}) +} + +func BenchmarkStorage_DeliveryZstd(b *testing.B) { + randomKey := "rcpt-" + strconv.FormatInt(time.Now().UnixNano(), 10) + "@example.org" + + be := createTestDB(b, "zstd") + if err := be.CreateIMAPAcct(randomKey); err != nil { + b.Fatal(err) + } + + testutils.BenchDelivery(b, be, "sender@example.org", []string{randomKey}) +} diff --git a/internal/storage/imapsql/delivery.go b/internal/storage/imapsql/delivery.go new file mode 100644 index 0000000..60cb2e1 --- /dev/null +++ b/internal/storage/imapsql/delivery.go @@ -0,0 +1,168 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imapsql + +import ( + "context" + "runtime/trace" + + "github.com/emersion/go-imap" + "github.com/emersion/go-imap/backend" + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + imapsql "github.com/foxcpp/go-imap-sql" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +type addedRcpt struct { + rcptTo string +} +type delivery struct { + store *Storage + msgMeta *module.MsgMetadata + d imapsql.Delivery + mailFrom string + + addedRcpts map[string]addedRcpt +} + +func (d *delivery) String() string { + return d.store.Name() + ":" + d.store.InstanceName() +} + +func userDoesNotExist(actual error) error { + return &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 1}, + Message: "User does not exist", + TargetName: "imapsql", + Err: actual, + } +} + +func (d *delivery) AddRcpt(ctx context.Context, rcptTo string, _ smtp.RcptOptions) error { + defer trace.StartRegion(ctx, "sql/AddRcpt").End() + + accountName, err := d.store.deliveryNormalize(ctx, rcptTo) + if err != nil { + return userDoesNotExist(err) + } + + if _, ok := d.addedRcpts[accountName]; ok { + return nil + } + + // This header is added to the message only for that recipient. + // go-imap-sql does certain optimizations to store the message + // with small amount of per-recipient data in a efficient way. + userHeader := textproto.Header{} + userHeader.Add("Delivered-To", accountName) + + if err := d.d.AddRcpt(accountName, userHeader); err != nil { + if err == imapsql.ErrUserDoesntExists || err == backend.ErrNoSuchMailbox { + return userDoesNotExist(err) + } + if _, ok := err.(imapsql.SerializationError); ok { + return &exterrors.SMTPError{ + Code: 453, + EnhancedCode: exterrors.EnhancedCode{4, 3, 2}, + Message: "Internal server error, try again later", + TargetName: "imapsql", + Err: err, + } + } + return err + } + + d.addedRcpts[accountName] = addedRcpt{ + rcptTo: rcptTo, + } + return nil +} + +func (d *delivery) Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error { + defer trace.StartRegion(ctx, "sql/Body").End() + + if !d.msgMeta.Quarantine && d.store.filters != nil { + for rcpt, rcptData := range d.addedRcpts { + folder, flags, err := d.store.filters.IMAPFilter(rcpt, rcptData.rcptTo, d.msgMeta, header, body) + if err != nil { + d.store.Log.Error("IMAPFilter failed", err, "rcpt", rcpt) + continue + } + d.d.UserMailbox(rcpt, folder, flags) + } + } + + if d.msgMeta.Quarantine { + if err := d.d.SpecialMailbox(imap.JunkAttr, d.store.junkMbox); err != nil { + if _, ok := err.(imapsql.SerializationError); ok { + return &exterrors.SMTPError{ + Code: 453, + EnhancedCode: exterrors.EnhancedCode{4, 3, 2}, + Message: "Storage access serialiation problem, try again later", + TargetName: "imapsql", + Err: err, + } + } + return err + } + } + + header = header.Copy() + header.Add("Return-Path", "<"+target.SanitizeForHeader(d.mailFrom)+">") + err := d.d.BodyParsed(header, body.Len(), body) + if _, ok := err.(imapsql.SerializationError); ok { + return &exterrors.SMTPError{ + Code: 453, + EnhancedCode: exterrors.EnhancedCode{4, 3, 2}, + Message: "Storage access serialiation problem, try again later", + TargetName: "imapsql", + Err: err, + } + } + return err +} + +func (d *delivery) Abort(ctx context.Context) error { + defer trace.StartRegion(ctx, "sql/Abort").End() + + return d.d.Abort() +} + +func (d *delivery) Commit(ctx context.Context) error { + defer trace.StartRegion(ctx, "sql/Commit").End() + + return d.d.Commit() +} + +func (store *Storage) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + defer trace.StartRegion(ctx, "sql/Start").End() + + return &delivery{ + store: store, + msgMeta: msgMeta, + mailFrom: mailFrom, + d: store.Back.NewDelivery(), + addedRcpts: map[string]addedRcpt{}, + }, nil +} diff --git a/internal/storage/imapsql/external_blob_store.go b/internal/storage/imapsql/external_blob_store.go new file mode 100644 index 0000000..0fb9cd7 --- /dev/null +++ b/internal/storage/imapsql/external_blob_store.go @@ -0,0 +1,68 @@ +package imapsql + +import ( + "context" + "io" + + imapsql "github.com/foxcpp/go-imap-sql" + "github.com/foxcpp/maddy/framework/module" +) + +type ExtBlob struct { + io.ReadCloser +} + +func (e ExtBlob) Sync() error { + panic("not implemented") +} + +func (e ExtBlob) Write(p []byte) (n int, err error) { + panic("not implemented") +} + +type WriteExtBlob struct { + module.Blob +} + +func (w WriteExtBlob) Read(p []byte) (n int, err error) { + panic("not implemented") +} + +type ExtBlobStore struct { + Base module.BlobStore +} + +func (e ExtBlobStore) Create(key string, objSize int64) (imapsql.ExtStoreObj, error) { + blob, err := e.Base.Create(context.TODO(), key, objSize) + if err != nil { + return nil, imapsql.ExternalError{ + NonExistent: err == module.ErrNoSuchBlob, + Key: key, + Err: err, + } + } + return WriteExtBlob{Blob: blob}, nil +} + +func (e ExtBlobStore) Open(key string) (imapsql.ExtStoreObj, error) { + blob, err := e.Base.Open(context.TODO(), key) + if err != nil { + return nil, imapsql.ExternalError{ + NonExistent: err == module.ErrNoSuchBlob, + Key: key, + Err: err, + } + } + return ExtBlob{ReadCloser: blob}, nil +} + +func (e ExtBlobStore) Delete(keys []string) error { + err := e.Base.Delete(context.TODO(), keys) + if err != nil { + return imapsql.ExternalError{ + Key: "", + Err: err, + } + } + return nil +} diff --git a/internal/storage/imapsql/imapsql.go b/internal/storage/imapsql/imapsql.go new file mode 100644 index 0000000..711a34e --- /dev/null +++ b/internal/storage/imapsql/imapsql.go @@ -0,0 +1,438 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package imapsql implements SQL-based storage module +// using go-imap-sql library (github.com/foxcpp/go-imap-sql). +// +// Interfaces implemented: +// - module.StorageBackend +// - module.PlainAuth +// - module.DeliveryTarget +package imapsql + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "path/filepath" + "runtime/debug" + "strconv" + "strings" + + "github.com/emersion/go-imap" + sortthread "github.com/emersion/go-imap-sortthread" + "github.com/emersion/go-imap/backend" + mess "github.com/foxcpp/go-imap-mess" + imapsql "github.com/foxcpp/go-imap-sql" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/authz" + "github.com/foxcpp/maddy/internal/updatepipe" + "github.com/foxcpp/maddy/internal/updatepipe/pubsub" + + _ "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" +) + +type Storage struct { + Back *imapsql.Backend + instName string + Log log.Logger + + junkMbox string + + driver string + dsn []string + + resolver dns.Resolver + + updPipe updatepipe.P + updPushStop chan struct{} + outboundUpds chan mess.Update + + filters module.IMAPFilter + + deliveryMap module.Table + deliveryNormalize func(context.Context, string) (string, error) + authMap module.Table + authNormalize func(context.Context, string) (string, error) +} + +func (store *Storage) Name() string { + return "imapsql" +} + +func (store *Storage) InstanceName() string { + return store.instName +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + store := &Storage{ + instName: instName, + Log: log.Logger{Name: "imapsql"}, + resolver: dns.DefaultResolver(), + } + if len(inlineArgs) != 0 { + if len(inlineArgs) == 1 { + return nil, errors.New("imapsql: expected at least 2 arguments") + } + + store.driver = inlineArgs[0] + store.dsn = inlineArgs[1:] + } + return store, nil +} + +func (store *Storage) Init(cfg *config.Map) error { + var ( + driver string + dsn []string + appendlimitVal int64 = -1 + compression []string + authNormalize string + deliveryNormalize string + + blobStore module.BlobStore + ) + + opts := imapsql.Opts{} + cfg.String("driver", false, false, store.driver, &driver) + cfg.StringList("dsn", false, false, store.dsn, &dsn) + cfg.Callback("fsstore", func(m *config.Map, node config.Node) error { + store.Log.Msg("'fsstore' directive is deprecated, use 'msg_store fs' instead") + return modconfig.ModuleFromNode("storage.blob", append([]string{"fs"}, node.Args...), + node, m.Globals, &blobStore) + }) + cfg.Custom("msg_store", false, false, func() (interface{}, error) { + var store module.BlobStore + err := modconfig.ModuleFromNode("storage.blob", []string{"fs", "messages"}, + config.Node{}, nil, &store) + return store, err + }, func(m *config.Map, node config.Node) (interface{}, error) { + var store module.BlobStore + err := modconfig.ModuleFromNode("storage.blob", node.Args, + node, m.Globals, &store) + return store, err + }, &blobStore) + cfg.StringList("compression", false, false, []string{"off"}, &compression) + cfg.DataSize("appendlimit", false, false, 32*1024*1024, &appendlimitVal) + cfg.Bool("debug", true, false, &store.Log.Debug) + cfg.Int("sqlite3_cache_size", false, false, 0, &opts.CacheSize) + cfg.Int("sqlite3_busy_timeout", false, false, 5000, &opts.BusyTimeout) + cfg.Bool("disable_recent", false, true, &opts.DisableRecent) + cfg.String("junk_mailbox", false, false, "Junk", &store.junkMbox) + cfg.Custom("imap_filter", false, false, func() (interface{}, error) { + return nil, nil + }, func(m *config.Map, node config.Node) (interface{}, error) { + var filter module.IMAPFilter + err := modconfig.GroupFromNode("imap_filters", node.Args, node, m.Globals, &filter) + return filter, err + }, &store.filters) + cfg.Custom("auth_map", false, false, func() (interface{}, error) { + return nil, nil + }, modconfig.TableDirective, &store.authMap) + cfg.String("auth_normalize", false, false, "auto", &authNormalize) + cfg.Custom("delivery_map", false, false, func() (interface{}, error) { + return nil, nil + }, modconfig.TableDirective, &store.deliveryMap) + cfg.String("delivery_normalize", false, false, "precis_casefold_email", &deliveryNormalize) + + if _, err := cfg.Process(); err != nil { + return err + } + + if dsn == nil { + return errors.New("imapsql: dsn is required") + } + if driver == "" { + return errors.New("imapsql: driver is required") + } + + if driver == "sqlite3" { + if sqliteImpl == "modernc" { + store.Log.Println("using transpiled SQLite (modernc.org/sqlite), this is experimental") + driver = "sqlite" + } else if sqliteImpl == "cgo" { + store.Log.Debugln("using cgo SQLite") + } else if sqliteImpl == "missing" { + return errors.New("imapsql: SQLite is not supported, recompile without no_sqlite3 tag set") + } + } + + deliveryNormFunc, ok := authz.NormalizeFuncs[deliveryNormalize] + if !ok { + return errors.New("imapsql: unknown normalization function: " + deliveryNormalize) + } + store.deliveryNormalize = func(ctx context.Context, s string) (string, error) { + return deliveryNormFunc(s) + } + if store.deliveryMap != nil { + store.deliveryNormalize = func(ctx context.Context, email string) (string, error) { + email, err := deliveryNormFunc(email) + if err != nil { + return "", err + } + mapped, ok, err := store.deliveryMap.Lookup(ctx, email) + if err != nil || !ok { + return "", userDoesNotExist(err) + } + return mapped, nil + } + } + + if authNormalize != "auto" { + store.Log.Msg("auth_normalize in storage.imapsql is deprecated and will be removed in the next release, use storage_map in imap config instead") + } + authNormFunc, ok := authz.NormalizeFuncs[authNormalize] + if !ok { + return errors.New("imapsql: unknown normalization function: " + authNormalize) + } + store.authNormalize = func(ctx context.Context, s string) (string, error) { + return authNormFunc(s) + } + if store.authMap != nil { + store.Log.Msg("auth_map in storage.imapsql is deprecated and will be removed in the next release, use storage_map in imap config instead") + store.authNormalize = func(ctx context.Context, username string) (string, error) { + username, err := authNormFunc(username) + if err != nil { + return "", err + } + mapped, ok, err := store.authMap.Lookup(ctx, username) + if err != nil || !ok { + return "", userDoesNotExist(err) + } + return mapped, nil + } + } + + opts.Log = &store.Log + + if appendlimitVal == -1 { + opts.MaxMsgBytes = nil + } else { + // int is 32-bit on some platforms, so cut off values we can't actually + // use. + if int64(uint32(appendlimitVal)) != appendlimitVal { + return errors.New("imapsql: appendlimit value is too big") + } + opts.MaxMsgBytes = new(uint32) + *opts.MaxMsgBytes = uint32(appendlimitVal) + } + var err error + + dsnStr := strings.Join(dsn, " ") + + if len(compression) != 0 { + switch compression[0] { + case "zstd", "lz4": + opts.CompressAlgo = compression[0] + if len(compression) == 2 { + opts.CompressAlgoParams = compression[1] + if _, err := strconv.Atoi(compression[1]); err != nil { + return errors.New("imapsql: first argument for lz4 and zstd is compression level") + } + } + if len(compression) > 2 { + return errors.New("imapsql: expected at most 2 arguments") + } + case "off": + if len(compression) > 1 { + return errors.New("imapsql: expected at most 1 arguments") + } + default: + return errors.New("imapsql: unknown compression algorithm") + } + } + + store.Back, err = imapsql.New(driver, dsnStr, ExtBlobStore{Base: blobStore}, opts) + if err != nil { + return fmt.Errorf("imapsql: %s", err) + } + + store.Log.Debugln("go-imap-sql version", imapsql.VersionStr) + + store.driver = driver + store.dsn = dsn + + return nil +} + +func (store *Storage) EnableUpdatePipe(mode updatepipe.BackendMode) error { + if store.updPipe != nil { + return nil + } + + switch store.driver { + case "sqlite3": + dbId := sha1.Sum([]byte(strings.Join(store.dsn, " "))) + sockPath := filepath.Join( + config.RuntimeDirectory, + fmt.Sprintf("sql-%s.sock", hex.EncodeToString(dbId[:]))) + store.Log.DebugMsg("using unix socket for external updates", "path", sockPath) + store.updPipe = &updatepipe.UnixSockPipe{ + SockPath: sockPath, + Log: log.Logger{Name: "storage.imapsql/updpipe", Debug: store.Log.Debug}, + } + case "postgres": + store.Log.DebugMsg("using PostgreSQL broker for external updates") + ps, err := pubsub.NewPQ(strings.Join(store.dsn, " ")) + if err != nil { + return fmt.Errorf("enable_update_pipe: %w", err) + } + ps.Log = log.Logger{Name: "storage.imapsql/updpipe/pubsub", Debug: store.Log.Debug} + pipe := &updatepipe.PubSubPipe{ + PubSub: ps, + Log: log.Logger{Name: "storage.imapsql/updpipe", Debug: store.Log.Debug}, + } + store.Back.UpdateManager().ExternalUnsubscribe = pipe.Unsubscribe + store.Back.UpdateManager().ExternalSubscribe = pipe.Subscribe + store.updPipe = pipe + default: + return errors.New("imapsql: driver does not have an update pipe implementation") + } + + inbound := make(chan mess.Update, 32) + outbound := make(chan mess.Update, 10) + store.outboundUpds = outbound + + if mode == updatepipe.ModeReplicate { + if err := store.updPipe.Listen(inbound); err != nil { + store.updPipe = nil + return err + } + } + + if err := store.updPipe.InitPush(); err != nil { + store.updPipe = nil + return err + } + + store.Back.UpdateManager().SetExternalSink(outbound) + + store.updPushStop = make(chan struct{}, 1) + go func() { + defer func() { + // Ensure we sent all outbound updates. + for upd := range outbound { + if err := store.updPipe.Push(upd); err != nil { + store.Log.Error("IMAP update pipe push failed", err) + } + } + store.updPushStop <- struct{}{} + + if err := recover(); err != nil { + stack := debug.Stack() + log.Printf("panic during imapsql update push: %v\n%s", err, stack) + } + }() + + for { + select { + case u := <-inbound: + store.Log.DebugMsg("external update received", "type", u.Type, "key", u.Key) + store.Back.UpdateManager().ExternalUpdate(u) + case u, ok := <-outbound: + if !ok { + return + } + store.Log.DebugMsg("sending external update", "type", u.Type, "key", u.Key) + if err := store.updPipe.Push(u); err != nil { + store.Log.Error("IMAP update pipe push failed", err) + } + } + } + }() + + return nil +} + +func (store *Storage) I18NLevel() int { + return 1 +} + +func (store *Storage) IMAPExtensions() []string { + return []string{"APPENDLIMIT", "MOVE", "CHILDREN", "SPECIAL-USE", "I18NLEVEL=1", "SORT", "THREAD=ORDEREDSUBJECT"} +} + +func (store *Storage) CreateMessageLimit() *uint32 { + return store.Back.CreateMessageLimit() +} + +func (store *Storage) GetOrCreateIMAPAcct(username string) (backend.User, error) { + accountName, err := store.authNormalize(context.TODO(), username) + if err != nil { + return nil, backend.ErrInvalidCredentials + } + + return store.Back.GetOrCreateUser(accountName) +} + +func (store *Storage) Lookup(ctx context.Context, key string) (string, bool, error) { + accountName, err := store.authNormalize(ctx, key) + if err != nil { + return "", false, nil + } + + usr, err := store.Back.GetUser(accountName) + if err != nil { + if errors.Is(err, imapsql.ErrUserDoesntExists) { + return "", false, nil + } + return "", false, err + } + if err := usr.Logout(); err != nil { + store.Log.Error("logout failed", err, "username", accountName) + } + + return "", true, nil +} + +func (store *Storage) Close() error { + // Stop backend from generating new updates. + store.Back.Close() + + // Wait for 'updates replicate' goroutine to actually stop so we will send + // all updates before shutting down (this is especially important for + // maddy subcommands). + if store.updPipe != nil { + close(store.outboundUpds) + <-store.updPushStop + + store.updPipe.Close() + } + + return nil +} + +func (store *Storage) Login(_ *imap.ConnInfo, usenrame, password string) (backend.User, error) { + panic("This method should not be called and is added only to satisfy backend.Backend interface") +} + +func (store *Storage) SupportedThreadAlgorithms() []sortthread.ThreadAlgorithm { + return []sortthread.ThreadAlgorithm{sortthread.OrderedSubject} +} + +func init() { + module.Register("storage.imapsql", New) + module.Register("target.imapsql", New) +} diff --git a/internal/storage/imapsql/maddyctl.go b/internal/storage/imapsql/maddyctl.go new file mode 100644 index 0000000..fa76638 --- /dev/null +++ b/internal/storage/imapsql/maddyctl.go @@ -0,0 +1,42 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imapsql + +import ( + "github.com/emersion/go-imap/backend" +) + +// These methods wrap corresponding go-imap-sql methods, but also apply +// maddy-specific credentials rules. + +func (store *Storage) ListIMAPAccts() ([]string, error) { + return store.Back.ListUsers() +} + +func (store *Storage) CreateIMAPAcct(accountName string) error { + return store.Back.CreateUser(accountName) +} + +func (store *Storage) DeleteIMAPAcct(accountName string) error { + return store.Back.DeleteUser(accountName) +} + +func (store *Storage) GetIMAPAcct(accountName string) (backend.User, error) { + return store.Back.GetUser(accountName) +} diff --git a/internal/storage/imapsql/modernc_sqlite3.go b/internal/storage/imapsql/modernc_sqlite3.go new file mode 100644 index 0000000..696b4c0 --- /dev/null +++ b/internal/storage/imapsql/modernc_sqlite3.go @@ -0,0 +1,26 @@ +//go:build !nosqlite3 && !cgo +// +build !nosqlite3,!cgo + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imapsql + +import _ "modernc.org/sqlite" + +const sqliteImpl = "modernc" diff --git a/internal/storage/imapsql/no_sqlite3.go b/internal/storage/imapsql/no_sqlite3.go new file mode 100644 index 0000000..525f8e4 --- /dev/null +++ b/internal/storage/imapsql/no_sqlite3.go @@ -0,0 +1,24 @@ +//go:build nosqlite3 +// +build nosqlite3 + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imapsql + +const sqliteImpl = "missing" diff --git a/internal/storage/imapsql/sqlite3.go b/internal/storage/imapsql/sqlite3.go new file mode 100644 index 0000000..599f39d --- /dev/null +++ b/internal/storage/imapsql/sqlite3.go @@ -0,0 +1,26 @@ +//go:build !nosqlite3 && cgo +// +build !nosqlite3,cgo + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package imapsql + +import _ "github.com/mattn/go-sqlite3" + +const sqliteImpl = "cgo" diff --git a/internal/table/chain.go b/internal/table/chain.go new file mode 100644 index 0000000..72eceba --- /dev/null +++ b/internal/table/chain.go @@ -0,0 +1,131 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/module" +) + +type Chain struct { + modName string + instName string + + chain []module.Table + optional []bool +} + +func NewChain(modName, instName string, _, _ []string) (module.Module, error) { + return &Chain{ + modName: modName, + instName: instName, + }, nil +} + +func (s *Chain) Init(cfg *config.Map) error { + cfg.Callback("step", func(m *config.Map, node config.Node) error { + var tbl module.Table + err := modconfig.ModuleFromNode("table", node.Args, node, m.Globals, &tbl) + if err != nil { + return err + } + + s.chain = append(s.chain, tbl) + s.optional = append(s.optional, false) + return nil + }) + cfg.Callback("optional_step", func(m *config.Map, node config.Node) error { + var tbl module.Table + err := modconfig.ModuleFromNode("table", node.Args, node, m.Globals, &tbl) + if err != nil { + return err + } + + s.chain = append(s.chain, tbl) + s.optional = append(s.optional, true) + return nil + }) + + _, err := cfg.Process() + return err +} + +func (s *Chain) Name() string { + return s.modName +} + +func (s *Chain) InstanceName() string { + return s.instName +} + +func (s *Chain) Lookup(ctx context.Context, key string) (string, bool, error) { + newVal, err := s.LookupMulti(ctx, key) + if err != nil { + return "", false, err + } + if len(newVal) == 0 { + return "", false, nil + } + + return newVal[0], true, nil +} + +func (s *Chain) LookupMulti(ctx context.Context, key string) ([]string, error) { + result := []string{key} +STEP: + for i, step := range s.chain { + newResult := []string{} + for _, key = range result { + if step_multi, ok := step.(module.MultiTable); ok { + val, err := step_multi.LookupMulti(ctx, key) + if err != nil { + return []string{}, err + } + if len(val) == 0 { + if s.optional[i] { + continue STEP + } + return []string{}, nil + } + newResult = append(newResult, val...) + } else { + val, ok, err := step.Lookup(ctx, key) + if err != nil { + return []string{}, err + } + if !ok { + if s.optional[i] { + continue STEP + } + return []string{}, nil + } + newResult = append(newResult, val) + } + } + result = newResult + } + return result, nil +} + +func init() { + module.Register("table.chain", NewChain) +} diff --git a/internal/table/email_localpart.go b/internal/table/email_localpart.go new file mode 100644 index 0000000..a9d6f06 --- /dev/null +++ b/internal/table/email_localpart.go @@ -0,0 +1,70 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +type EmailLocalpart struct { + modName string + instName string + allowNonEmail bool +} + +func NewEmailLocalpart(modName, instName string, _, _ []string) (module.Module, error) { + return &EmailLocalpart{ + modName: modName, + instName: instName, + allowNonEmail: modName == "table.email_localpart_optional", + }, nil +} + +func (s *EmailLocalpart) Init(cfg *config.Map) error { + return nil +} + +func (s *EmailLocalpart) Name() string { + return s.modName +} + +func (s *EmailLocalpart) InstanceName() string { + return s.modName +} + +func (s *EmailLocalpart) Lookup(ctx context.Context, key string) (string, bool, error) { + mbox, _, err := address.Split(key) + if err != nil { + if s.allowNonEmail { + return key, true, nil + } + // Invalid email, no local part mapping. + return "", false, nil + } + return mbox, true, nil +} + +func init() { + module.Register("table.email_localpart", NewEmailLocalpart) + module.Register("table.email_localpart_optional", NewEmailLocalpart) +} diff --git a/internal/table/email_with_domain.go b/internal/table/email_with_domain.go new file mode 100644 index 0000000..4eb50b5 --- /dev/null +++ b/internal/table/email_with_domain.go @@ -0,0 +1,89 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + "fmt" + + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +type EmailWithDomain struct { + modName string + instName string + domains []string + log log.Logger +} + +func NewEmailWithDomain(modName, instName string, _, inlineArgs []string) (module.Module, error) { + return &EmailWithDomain{ + modName: modName, + instName: instName, + domains: inlineArgs, + log: log.Logger{Name: modName}, + }, nil +} + +func (s *EmailWithDomain) Init(cfg *config.Map) error { + for _, d := range s.domains { + if !address.ValidDomain(d) { + return fmt.Errorf("%s: invalid domain: %s", s.modName, d) + } + } + if len(s.domains) == 0 { + return fmt.Errorf("%s: at least one domain is required", s.modName) + } + + return nil +} + +func (s *EmailWithDomain) Name() string { + return s.modName +} + +func (s *EmailWithDomain) InstanceName() string { + return s.modName +} + +func (s *EmailWithDomain) Lookup(ctx context.Context, key string) (string, bool, error) { + quotedMbox := address.QuoteMbox(key) + + if len(s.domains) == 0 { + s.log.Msg("only first domain is used when expanding key", "key", key, "domain", s.domains[0]) + } + + return quotedMbox + "@" + s.domains[0], true, nil +} + +func (s *EmailWithDomain) LookupMulti(ctx context.Context, key string) ([]string, error) { + quotedMbox := address.QuoteMbox(key) + emails := make([]string, len(s.domains)) + for i, domain := range s.domains { + emails[i] = quotedMbox + "@" + domain + } + return emails, nil +} + +func init() { + module.Register("table.email_with_domain", NewEmailWithDomain) +} diff --git a/internal/table/file.go b/internal/table/file.go new file mode 100644 index 0000000..a0286a9 --- /dev/null +++ b/internal/table/file.go @@ -0,0 +1,258 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "bufio" + "context" + "fmt" + "os" + "runtime/debug" + "strings" + "sync" + "time" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +const FileModName = "table.file" + +type File struct { + instName string + file string + + m map[string][]string + mLck sync.RWMutex + mStamp time.Time + + stopReloader chan struct{} + forceReload chan struct{} + + log log.Logger +} + +func NewFile(_, instName string, _, inlineArgs []string) (module.Module, error) { + m := &File{ + instName: instName, + m: make(map[string][]string), + stopReloader: make(chan struct{}), + forceReload: make(chan struct{}), + log: log.Logger{Name: FileModName}, + } + + switch len(inlineArgs) { + case 1: + m.file = inlineArgs[0] + case 0: + default: + return nil, fmt.Errorf("%s: cannot use multiple files with single %s, use %s multiple times to do so", FileModName, FileModName, FileModName) + } + + return m, nil +} + +func (f *File) Name() string { + return FileModName +} + +func (f *File) InstanceName() string { + return f.instName +} + +func (f *File) Init(cfg *config.Map) error { + var file string + cfg.Bool("debug", true, false, &f.log.Debug) + cfg.String("file", false, false, "", &file) + if _, err := cfg.Process(); err != nil { + return err + } + + if file != "" { + if f.file != "" { + return fmt.Errorf("%s: file path specified both in directive and in argument, do it once", FileModName) + } + f.file = file + } + + if err := readFile(f.file, f.m); err != nil { + if !os.IsNotExist(err) { + return err + } + f.log.Printf("ignoring non-existent file: %s", f.file) + } + + go f.reloader() + hooks.AddHook(hooks.EventReload, func() { + f.forceReload <- struct{}{} + }) + + return nil +} + +var reloadInterval = 15 * time.Second + +func (f *File) reloader() { + defer func() { + if err := recover(); err != nil { + stack := debug.Stack() + log.Printf("panic during m reload: %v\n%s", err, stack) + } + }() + + t := time.NewTicker(reloadInterval) + defer t.Stop() + + for { + select { + case <-t.C: + f.reload() + + case <-f.forceReload: + f.reload() + + case <-f.stopReloader: + f.stopReloader <- struct{}{} + return + } + } +} + +func (f *File) reload() { + info, err := os.Stat(f.file) + if err != nil { + if os.IsNotExist(err) { + f.mLck.Lock() + f.m = map[string][]string{} + f.mLck.Unlock() + return + } + f.log.Error("os stat", err) + } + if info.ModTime().Before(f.mStamp) || time.Since(info.ModTime()) < (reloadInterval/2) { + return // reload not necessary + } + + f.log.Debugf("reloading") + + newm := make(map[string][]string, len(f.m)+5) + if err := readFile(f.file, newm); err != nil { + if os.IsNotExist(err) { + f.log.Printf("ignoring non-existent file: %s", f.file) + return + } + + f.log.Println(err) + return + } + // after reading we need to check whether file has changed in between + info2, err := os.Stat(f.file) + if err != nil { + f.log.Println(err) + return + } + + if !info2.ModTime().Equal(info.ModTime()) { + // file has changed in the meantime + return + } + + f.mLck.Lock() + f.m = newm + f.mStamp = info.ModTime() + f.mLck.Unlock() +} + +func (f *File) Close() error { + f.stopReloader <- struct{}{} + <-f.stopReloader + return nil +} + +func readFile(path string, out map[string][]string) error { + f, err := os.Open(path) + if err != nil { + return err + } + + scnr := bufio.NewScanner(f) + lineCounter := 0 + + parseErr := func(text string) error { + return fmt.Errorf("%s:%d: %s", path, lineCounter, text) + } + + for scnr.Scan() { + lineCounter++ + if strings.HasPrefix(scnr.Text(), "#") { + continue + } + + text := strings.TrimSpace(scnr.Text()) + if text == "" { + continue + } + + parts := strings.SplitN(text, ":", 2) + if len(parts) == 1 { + parts = append(parts, "") + } + + from := strings.TrimSpace(parts[0]) + if len(from) == 0 { + return parseErr("empty address before colon") + } + + for _, to := range strings.Split(parts[1], ",") { + to := strings.TrimSpace(to) + out[from] = append(out[from], to) + } + } + return scnr.Err() +} + +func (f *File) Lookup(_ context.Context, val string) (string, bool, error) { + // The existing map is never modified, instead it is replaced with a new + // one if reload is performed. + f.mLck.RLock() + usedFile := f.m + f.mLck.RUnlock() + + newVal, ok := usedFile[val] + + if len(newVal) == 0 { + return "", false, nil + } + + return newVal[0], ok, nil +} + +func (f *File) LookupMulti(_ context.Context, val string) ([]string, error) { + f.mLck.RLock() + usedFile := f.m + f.mLck.RUnlock() + + return usedFile[val], nil +} + +func init() { + module.Register(FileModName, NewFile) +} diff --git a/internal/table/file_test.go b/internal/table/file_test.go new file mode 100644 index 0000000..c51620d --- /dev/null +++ b/internal/table/file_test.go @@ -0,0 +1,222 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "os" + "reflect" + "testing" + "time" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestReadFile(t *testing.T) { + test := func(file string, expected map[string][]string) { + t.Helper() + + f, err := os.CreateTemp("", "maddy-tests-") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + defer f.Close() + if _, err := f.WriteString(file); err != nil { + t.Fatal(err) + } + + actual := map[string][]string{} + err = readFile(f.Name(), actual) + if expected == nil { + if err == nil { + t.Errorf("expected failure, got %+v", actual) + } + return + } + if err != nil { + t.Errorf("unexpected failure: %v", err) + return + } + + if !reflect.DeepEqual(actual, expected) { + t.Errorf("wrong results\n want %+v\n got %+v", expected, actual) + } + } + + test("a: b", map[string][]string{"a": {"b"}}) + test("a@example.org: b@example.com", map[string][]string{"a@example.org": {"b@example.com"}}) + test(`"a @ a"@example.org: b@example.com`, map[string][]string{`"a @ a"@example.org`: {"b@example.com"}}) + test(`a@example.org: "b @ b"@example.com`, map[string][]string{`a@example.org`: {`"b @ b"@example.com`}}) + test(`"a @ a": "b @ b"`, map[string][]string{`"a @ a"`: {`"b @ b"`}}) + test("a: b, c", map[string][]string{"a": {"b", "c"}}) + test("a: b\na: c", map[string][]string{"a": {"b", "c"}}) + test(": b", nil) + test(":", nil) + test("aaa", map[string][]string{"aaa": {""}}) + test(": b", nil) + test(" testing@example.com : arbitrary-whitespace@example.org ", + map[string][]string{"testing@example.com": {"arbitrary-whitespace@example.org"}}) + test(`# skip comments +a: b`, map[string][]string{"a": {"b"}}) + test(`# and empty lines + +a: b`, map[string][]string{"a": {"b"}}) + test("# with whitespace too\n \na: b", map[string][]string{"a": {"b"}}) + test("a: b\na: c", map[string][]string{"a": {"b", "c"}}) +} + +func TestFileReload(t *testing.T) { + t.Parallel() + + const file = `cat: dog` + + f, err := os.CreateTemp("", "maddy-tests-") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + if _, err := f.WriteString(file); err != nil { + f.Close() + t.Fatal(err) + } + f.Close() + + mod, err := NewFile("", "", nil, []string{f.Name()}) + if err != nil { + t.Fatal(err) + } + m := mod.(*File) + m.log = testutils.Logger(t, "file_map") + defer m.Close() + + if err := mod.Init(&config.Map{Block: config.Node{}}); err != nil { + t.Fatal(err) + } + + // ensure it is correctly loaded at first time. + m.mLck.RLock() + if m.m["cat"] == nil { + t.Fatalf("wrong content loaded, new m were not loaded, %v", m.m) + } + m.mLck.RUnlock() + + for i := 0; i < 100; i++ { + // try to provoke race condition on file writing + if i%2 == 0 { + if err := os.WriteFile(f.Name(), []byte("dog: cat"), os.ModePerm); err != nil { + t.Fatal(err) + } + } + time.Sleep(reloadInterval + 5*time.Millisecond) + m.mLck.RLock() + if m.m["dog"] == nil { + t.Fatalf("wrong content loaded, new m were not loaded, %v", m.m) + } + m.mLck.RUnlock() + } +} + +func TestFileReload_Broken(t *testing.T) { + t.Parallel() + + const file = `cat: dog` + + f, err := os.CreateTemp("", "maddy-tests-") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + if _, err := f.WriteString(file); err != nil { + f.Close() + t.Fatal(err) + } + f.Close() + + mod, err := NewFile("", "", nil, []string{f.Name()}) + if err != nil { + t.Fatal(err) + } + m := mod.(*File) + m.log = testutils.Logger(t, FileModName) + defer m.Close() + + if err := mod.Init(&config.Map{Block: config.Node{}}); err != nil { + t.Fatal(err) + } + + f2, err := os.OpenFile(f.Name(), os.O_WRONLY|os.O_SYNC, os.ModePerm) + if err != nil { + t.Fatal(err) + } + if _, err := f2.WriteString(":"); err != nil { + t.Fatal(err) + } + defer f2.Close() + + time.Sleep(3 * reloadInterval) + + m.mLck.RLock() + defer m.mLck.RUnlock() + if m.m["cat"] == nil { + t.Fatal("New m were loaded or map changed", m.m) + } +} + +func TestFileReload_Removed(t *testing.T) { + t.Parallel() + + const file = `cat: dog` + + f, err := os.CreateTemp("", "maddy-tests-") + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteString(file); err != nil { + f.Close() + t.Fatal(err) + } + f.Close() + + mod, err := NewFile("", "", nil, []string{f.Name()}) + if err != nil { + t.Fatal(err) + } + m := mod.(*File) + m.log = testutils.Logger(t, FileModName) + defer m.Close() + + if err := mod.Init(&config.Map{Block: config.Node{}}); err != nil { + t.Fatal(err) + } + + os.Remove(f.Name()) + + time.Sleep(3 * reloadInterval) + + m.mLck.RLock() + defer m.mLck.RUnlock() + if m.m["cat"] != nil { + t.Fatal("Old m are still loaded") + } +} + +func init() { + reloadInterval = 10 * time.Millisecond +} diff --git a/internal/table/identity.go b/internal/table/identity.go new file mode 100644 index 0000000..c405d3d --- /dev/null +++ b/internal/table/identity.go @@ -0,0 +1,58 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +type Identity struct { + modName string + instName string +} + +func NewIdentity(modName, instName string, _, _ []string) (module.Module, error) { + return &Identity{ + modName: modName, + instName: instName, + }, nil +} + +func (s *Identity) Init(cfg *config.Map) error { + return nil +} + +func (s *Identity) Name() string { + return s.modName +} + +func (s *Identity) InstanceName() string { + return s.modName +} + +func (s *Identity) Lookup(_ context.Context, key string) (string, bool, error) { + return key, true, nil +} + +func init() { + module.Register("table.identity", NewIdentity) +} diff --git a/internal/table/regexp.go b/internal/table/regexp.go new file mode 100644 index 0000000..069be6c --- /dev/null +++ b/internal/table/regexp.go @@ -0,0 +1,127 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + "fmt" + "regexp" + "strings" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +type Regexp struct { + modName string + instName string + inlineArgs []string + + re *regexp.Regexp + replacements []string + + expandPlaceholders bool +} + +func NewRegexp(modName, instName string, _, inlineArgs []string) (module.Module, error) { + return &Regexp{ + modName: modName, + instName: instName, + inlineArgs: inlineArgs, + }, nil +} + +func (r *Regexp) Init(cfg *config.Map) error { + var ( + fullMatch bool + caseInsensitive bool + ) + cfg.Bool("full_match", false, true, &fullMatch) + cfg.Bool("case_insensitive", false, true, &caseInsensitive) + cfg.Bool("expand_replaceholders", false, true, &r.expandPlaceholders) + if _, err := cfg.Process(); err != nil { + return err + } + + regex := r.inlineArgs[0] + if len(r.inlineArgs) > 1 { + r.replacements = r.inlineArgs[1:] + } + + if fullMatch { + if !strings.HasPrefix(regex, "^") { + regex = "^" + regex + } + if !strings.HasSuffix(regex, "$") { + regex = regex + "$" + } + } + + if caseInsensitive { + regex = "(?i)" + regex + } + + var err error + r.re, err = regexp.Compile(regex) + if err != nil { + return fmt.Errorf("%s: %v", r.modName, err) + } + return nil +} + +func (r *Regexp) Name() string { + return r.modName +} + +func (r *Regexp) InstanceName() string { + return r.modName +} + +func (r *Regexp) LookupMulti(_ context.Context, key string) ([]string, error) { + matches := r.re.FindStringSubmatchIndex(key) + if matches == nil { + return []string{}, nil + } + + result := []string{} + for _, replacement := range r.replacements { + if !r.expandPlaceholders { + result = append(result, replacement) + } else { + result = append(result, string(r.re.ExpandString([]byte{}, replacement, key, matches))) + } + } + return result, nil +} + +func (r *Regexp) Lookup(ctx context.Context, key string) (string, bool, error) { + newVal, err := r.LookupMulti(ctx, key) + if err != nil { + return "", false, err + } + if len(newVal) == 0 { + return "", false, nil + } + + return newVal[0], true, nil +} + +func init() { + module.Register("table.regexp", NewRegexp) +} diff --git a/internal/table/sql_query.go b/internal/table/sql_query.go new file mode 100644 index 0000000..c15f710 --- /dev/null +++ b/internal/table/sql_query.go @@ -0,0 +1,251 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + _ "github.com/lib/pq" +) + +type SQL struct { + modName string + instName string + + namedArgs bool + + db *sql.DB + lookup *sql.Stmt + add *sql.Stmt + list *sql.Stmt + set *sql.Stmt + del *sql.Stmt +} + +func NewSQL(modName, instName string, _, _ []string) (module.Module, error) { + return &SQL{ + modName: modName, + instName: instName, + }, nil +} + +func (s *SQL) Name() string { + return s.modName +} + +func (s *SQL) InstanceName() string { + return s.instName +} + +func (s *SQL) Init(cfg *config.Map) error { + var ( + driver string + initQueries []string + dsnParts []string + lookupQuery string + + addQuery string + listQuery string + removeQuery string + setQuery string + ) + cfg.StringList("init", false, false, nil, &initQueries) + cfg.String("driver", false, true, "", &driver) + cfg.StringList("dsn", false, true, nil, &dsnParts) + cfg.Bool("named_args", false, false, &s.namedArgs) + + cfg.String("lookup", false, true, "", &lookupQuery) + + cfg.String("add", false, false, "", &addQuery) + cfg.String("list", false, false, "", &listQuery) + cfg.String("del", false, false, "", &removeQuery) + cfg.String("set", false, false, "", &setQuery) + if _, err := cfg.Process(); err != nil { + return err + } + + if driver == "postgres" && s.namedArgs { + return config.NodeErr(cfg.Block, "PostgreSQL driver does not support named_args") + } + + db, err := sql.Open(driver, strings.Join(dsnParts, " ")) + if err != nil { + return config.NodeErr(cfg.Block, "failed to open db: %v", err) + } + s.db = db + + for _, init := range initQueries { + if _, err := db.Exec(init); err != nil { + return config.NodeErr(cfg.Block, "init query failed: %v", err) + } + } + + s.lookup, err = db.Prepare(lookupQuery) + if err != nil { + return config.NodeErr(cfg.Block, "failed to prepare lookup query: %v", err) + } + if addQuery != "" { + s.add, err = db.Prepare(addQuery) + if err != nil { + return config.NodeErr(cfg.Block, "failed to prepare add query: %v", err) + } + } + if listQuery != "" { + s.list, err = db.Prepare(listQuery) + if err != nil { + return config.NodeErr(cfg.Block, "failed to prepare list query: %v", err) + } + } + if setQuery != "" { + s.set, err = db.Prepare(setQuery) + if err != nil { + return config.NodeErr(cfg.Block, "failed to prepare set query: %v", err) + } + } + if removeQuery != "" { + s.del, err = db.Prepare(removeQuery) + if err != nil { + return config.NodeErr(cfg.Block, "failed to prepare del query: %v", err) + } + } + + return nil +} + +func (s *SQL) Close() error { + s.lookup.Close() + return s.db.Close() +} + +func (s *SQL) Lookup(ctx context.Context, val string) (string, bool, error) { + var ( + repl string + row *sql.Row + ) + if s.namedArgs { + row = s.lookup.QueryRowContext(ctx, sql.Named("key", val)) + } else { + row = s.lookup.QueryRowContext(ctx, val) + } + if err := row.Scan(&repl); err != nil { + if err == sql.ErrNoRows { + return "", false, nil + } + return "", false, fmt.Errorf("%s: lookup %s: %w", s.modName, val, err) + } + return repl, true, nil +} + +func (s *SQL) LookupMulti(ctx context.Context, val string) ([]string, error) { + var ( + repl []string + rows *sql.Rows + err error + ) + if s.namedArgs { + rows, err = s.lookup.QueryContext(ctx, sql.Named("key", val)) + } else { + rows, err = s.lookup.QueryContext(ctx, val) + } + if err != nil { + return nil, fmt.Errorf("%s; lookup %s: %w", s.modName, val, err) + } + for rows.Next() { + var res string + if err := rows.Scan(&res); err != nil { + return nil, fmt.Errorf("%s; lookup %s: %w", s.modName, val, err) + } + repl = append(repl, res) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("%s; lookup %s: %w", s.modName, val, err) + } + return repl, nil +} + +func (s *SQL) Keys() ([]string, error) { + if s.list == nil { + return nil, fmt.Errorf("%s: table is not mutable (no 'list' query)", s.modName) + } + + rows, err := s.list.Query() + if err != nil { + return nil, fmt.Errorf("%s: list: %w", s.modName, err) + } + defer rows.Close() + var list []string + for rows.Next() { + var key string + if err := rows.Scan(&key); err != nil { + return nil, fmt.Errorf("%s: list: %w", s.modName, err) + } + list = append(list, key) + } + return list, nil +} + +func (s *SQL) RemoveKey(k string) error { + if s.del == nil { + return fmt.Errorf("%s: table is not mutable (no 'del' query)", s.modName) + } + + var err error + if s.namedArgs { + _, err = s.del.Exec(sql.Named("key", k)) + } else { + _, err = s.del.Exec(k) + } + if err != nil { + return fmt.Errorf("%s: del %s: %w", s.modName, k, err) + } + return nil +} + +func (s *SQL) SetKey(k, v string) error { + if s.set == nil { + return fmt.Errorf("%s: table is not mutable (no 'set' query)", s.modName) + } + if s.add == nil { + return fmt.Errorf("%s: table is not mutable (no 'add' query)", s.modName) + } + + var args []interface{} + if s.namedArgs { + args = []interface{}{sql.Named("key", k), sql.Named("value", v)} + } else { + args = []interface{}{k, v} + } + + if _, err := s.add.Exec(args...); err != nil { + if _, err := s.set.Exec(args...); err != nil { + return fmt.Errorf("%s: add %s: %w", s.modName, k, err) + } + return nil + } + return nil +} + +func init() { + module.Register("table.sql_query", NewSQL) +} diff --git a/internal/table/sql_query_test.go b/internal/table/sql_query_test.go new file mode 100644 index 0000000..fd160f7 --- /dev/null +++ b/internal/table/sql_query_test.go @@ -0,0 +1,96 @@ +//go:build !nosqlite3 && cgo +// +build !nosqlite3,cgo + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + "path/filepath" + "reflect" + "testing" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestSQL(t *testing.T) { + path := testutils.Dir(t) + mod, err := NewSQL("sql_table", "", nil, nil) + if err != nil { + t.Fatal("Module create failed:", err) + } + tbl := mod.(*SQL) + err = tbl.Init(config.NewMap(nil, config.Node{ + Children: []config.Node{ + { + Name: "driver", + Args: []string{"sqlite3"}, + }, + { + Name: "dsn", + Args: []string{filepath.Join(path, "test.db")}, + }, + { + Name: "init", + Args: []string{ + "CREATE TABLE testTbl (key TEXT, value TEXT)", + "INSERT INTO testTbl VALUES ('user1', 'user1a')", + "INSERT INTO testTbl VALUES ('user1', 'user1b')", + "INSERT INTO testTbl VALUES ('user3', NULL)", + }, + }, + { + Name: "lookup", + Args: []string{"SELECT value FROM testTbl WHERE key = $key"}, + }, + }, + })) + if err != nil { + t.Fatal("Init failed:", err) + } + + check := func(key, res string, ok, fail bool) { + t.Helper() + + actualRes, actualOk, err := tbl.Lookup(context.Background(), key) + if actualRes != res { + t.Errorf("Result mismatch: want %s, got %s", res, actualRes) + } + if actualOk != ok { + t.Errorf("OK mismatch: want %v, got %v", actualOk, ok) + } + if (err != nil) != fail { + t.Errorf("Error mismatch: want failure = %v, got %v", fail, err) + } + } + + check("user1", "user1a", true, false) + check("user2", "", false, false) + check("user3", "", false, true) + + vals, err := tbl.LookupMulti(context.Background(), "user1") + if err != nil { + t.Error("Unexpected error:", err) + } + if !reflect.DeepEqual(vals, []string{"user1a", "user1b"}) { + t.Error("Wrong result of LookupMulti:", vals) + } +} diff --git a/internal/table/sql_table.go b/internal/table/sql_table.go new file mode 100644 index 0000000..e793dc4 --- /dev/null +++ b/internal/table/sql_table.go @@ -0,0 +1,173 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + "fmt" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + _ "github.com/lib/pq" +) + +type SQLTable struct { + modName string + instName string + + wrapped *SQL +} + +func NewSQLTable(modName, instName string, _, _ []string) (module.Module, error) { + return &SQLTable{ + modName: modName, + instName: instName, + + wrapped: &SQL{ + modName: modName, + instName: instName, + }, + }, nil +} + +func (s *SQLTable) Name() string { + return s.modName +} + +func (s *SQLTable) InstanceName() string { + return s.instName +} + +func (s *SQLTable) Init(cfg *config.Map) error { + var ( + driver string + dsnParts []string + tableName string + keyColumn string + valueColumn string + ) + cfg.String("driver", false, true, "", &driver) + cfg.StringList("dsn", false, true, nil, &dsnParts) + cfg.String("table_name", false, true, "", &tableName) + cfg.String("key_column", false, false, "key", &keyColumn) + cfg.String("value_column", false, false, "value", &valueColumn) + if _, err := cfg.Process(); err != nil { + return err + } + + // sql_table module literally wraps the sql_query module by generating a + // configuration block for it. + + var ( + useNamedArgs string + + lookupQuery string + addQuery string + listQuery string + setQuery string + delQuery string + ) + if driver == "sqlite3" { + useNamedArgs = "yes" + lookupQuery = fmt.Sprintf("SELECT %s FROM %s WHERE %s = :key", valueColumn, tableName, keyColumn) + addQuery = fmt.Sprintf("INSERT INTO %s(%s, %s) VALUES(:key, :value)", tableName, keyColumn, valueColumn) + listQuery = fmt.Sprintf("SELECT %s from %s", keyColumn, tableName) + setQuery = fmt.Sprintf("UPDATE %s SET %s = :value WHERE %s = :key", tableName, valueColumn, keyColumn) + delQuery = fmt.Sprintf("DELETE FROM %s WHERE %s = :key", tableName, keyColumn) + } else { + useNamedArgs = "no" + lookupQuery = fmt.Sprintf("SELECT %s FROM %s WHERE %s = $1", valueColumn, tableName, keyColumn) + addQuery = fmt.Sprintf("INSERT INTO %s(%s, %s) VALUES($1, $2)", tableName, keyColumn, valueColumn) + listQuery = fmt.Sprintf("SELECT %s from %s", keyColumn, tableName) + setQuery = fmt.Sprintf("UPDATE %s SET %s = $2 WHERE %s = $1", tableName, valueColumn, keyColumn) + delQuery = fmt.Sprintf("DELETE FROM %s WHERE %s = $1", tableName, keyColumn) + } + + return s.wrapped.Init(config.NewMap(cfg.Globals, config.Node{ + Children: []config.Node{ + { + Name: "driver", + Args: []string{driver}, + }, + { + Name: "dsn", + Args: dsnParts, + }, + { + Name: "named_args", + Args: []string{useNamedArgs}, + }, + { + Name: "lookup", + Args: []string{lookupQuery}, + }, + { + Name: "add", + Args: []string{addQuery}, + }, + { + Name: "list", + Args: []string{listQuery}, + }, + { + Name: "set", + Args: []string{setQuery}, + }, + { + Name: "del", + Args: []string{delQuery}, + }, + { + Name: "init", + Args: []string{fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( + %s TEXT PRIMARY KEY NOT NULL, + %s TEXT NOT NULL + )`, tableName, keyColumn, valueColumn)}, + }, + }, + })) +} + +func (s *SQLTable) Close() error { + return s.wrapped.Close() +} + +func (s *SQLTable) Lookup(ctx context.Context, val string) (string, bool, error) { + return s.wrapped.Lookup(ctx, val) +} + +func (s *SQLTable) LookupMulti(ctx context.Context, val string) ([]string, error) { + return s.wrapped.LookupMulti(ctx, val) +} + +func (s *SQLTable) Keys() ([]string, error) { + return s.wrapped.Keys() +} + +func (s *SQLTable) RemoveKey(k string) error { + return s.wrapped.RemoveKey(k) +} + +func (s *SQLTable) SetKey(k, v string) error { + return s.wrapped.SetKey(k, v) +} + +func init() { + module.Register("table.sql_table", NewSQLTable) +} diff --git a/internal/table/sqlite3.go b/internal/table/sqlite3.go new file mode 100644 index 0000000..8b22794 --- /dev/null +++ b/internal/table/sqlite3.go @@ -0,0 +1,24 @@ +//go:build !nosqlite3 && cgo +// +build !nosqlite3,cgo + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import _ "github.com/mattn/go-sqlite3" diff --git a/internal/table/static.go b/internal/table/static.go new file mode 100644 index 0000000..e55ffd5 --- /dev/null +++ b/internal/table/static.go @@ -0,0 +1,77 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package table + +import ( + "context" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +type Static struct { + modName string + instName string + + m map[string][]string +} + +func NewStatic(modName, instName string, _, _ []string) (module.Module, error) { + return &Static{ + modName: modName, + instName: instName, + m: map[string][]string{}, + }, nil +} + +func (s *Static) Init(cfg *config.Map) error { + cfg.Callback("entry", func(_ *config.Map, node config.Node) error { + if len(node.Args) < 2 { + return config.NodeErr(node, "expected at least one value") + } + s.m[node.Args[0]] = node.Args[1:] + return nil + }) + _, err := cfg.Process() + return err +} + +func (s *Static) Name() string { + return s.modName +} + +func (s *Static) InstanceName() string { + return s.modName +} + +func (s *Static) Lookup(ctx context.Context, key string) (string, bool, error) { + val := s.m[key] + if len(val) == 0 { + return "", false, nil + } + return val[0], true, nil +} + +func (s *Static) LookupMulti(ctx context.Context, key string) ([]string, error) { + return s.m[key], nil +} + +func init() { + module.Register("table.static", NewStatic) +} diff --git a/internal/target/delivery.go b/internal/target/delivery.go new file mode 100644 index 0000000..1c3450f --- /dev/null +++ b/internal/target/delivery.go @@ -0,0 +1,34 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package target + +import ( + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +func DeliveryLogger(l log.Logger, msgMeta *module.MsgMetadata) log.Logger { + fields := make(map[string]interface{}, len(l.Fields)+1) + for k, v := range l.Fields { + fields[k] = v + } + fields["msg_id"] = msgMeta.ID + l.Fields = fields + return l +} diff --git a/internal/target/queue/metrics.go b/internal/target/queue/metrics.go new file mode 100644 index 0000000..32ada20 --- /dev/null +++ b/internal/target/queue/metrics.go @@ -0,0 +1,35 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package queue + +import "github.com/prometheus/client_golang/prometheus" + +var queuedMsgs = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "maddy", + Subsystem: "queue", + Name: "length", + Help: "Amount of queued messages", + }, + []string{"module", "location"}, +) + +func init() { + prometheus.MustRegister(queuedMsgs) +} diff --git a/internal/target/queue/queue.go b/internal/target/queue/queue.go new file mode 100644 index 0000000..264a70f --- /dev/null +++ b/internal/target/queue/queue.go @@ -0,0 +1,998 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +/* +Package queue implements module which keeps messages on disk and tries delivery +to the configured target (usually remote) multiple times until all recipients +are succeeded. + +Interfaces implemented: +- module.DeliveryTarget + +Implementation summary follows. + +All scheduled deliveries are attempted to the configured DeliveryTarget. +All metadata is preserved on disk. + +Failure status is determined on per-recipient basis: + - Delivery.Start fail handled as a failure for all recipients. + - Delivery.AddRcpt fail handled as a failure for the corresponding recipient. + - Delivery.Body fail handled as a failure for all recipients. + - If Delivery implements PartialDelivery, then + PartialDelivery.BodyNonAtomic is used instead. Failures are determined based + on StatusCollector.SetStatus calls done by target in this case. + +For each failure check is done to see if it is a permanent failure +or a temporary one. This is done using exterrors.IsTemporaryOrUnspec. +That is, errors are assumed to be temporary by default. +All errors are converted to SMTPError then due to a storage limitations. + +If there are any *temporary* failed recipients, delivery will be retried +after delay *only for these* recipients. + +Last error for each recipient is saved for reporting in NDN. A NDN is generated +if there are any failed recipients left after +last attempt to deliver the message. + +Amount of attempts for each message is limited to a certain configured number. +After last attempt, all recipients that are still temporary failing are assumed +to be permanently failed. +*/ +package queue + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "runtime" + "runtime/debug" + "runtime/trace" + "strconv" + "strings" + "sync" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/dsn" + "github.com/foxcpp/maddy/internal/msgpipeline" + "github.com/foxcpp/maddy/internal/target" +) + +// partialError describes state of partially successful message delivery. +type partialError struct { + + // Underlying error objects for each recipient. + Errs map[string]error + + // Fields can be accessed without holding this lock, but only after + // target.BodyNonAtomic/Body returns. + statusLock *sync.Mutex +} + +// SetStatus implements module.StatusCollector so partialError can be +// passed directly to PartialDelivery.BodyNonAtomic. +func (pe *partialError) SetStatus(rcptTo string, err error) { + log.Debugf("PartialError.SetStatus(%s, %v)", rcptTo, err) + if err == nil { + return + } + pe.statusLock.Lock() + defer pe.statusLock.Unlock() + pe.Errs[rcptTo] = err +} + +func (pe partialError) Error() string { + return fmt.Sprintf("delivery failed for some recipients: %v", pe.Errs) +} + +// dontRecover controls the behavior of panic handlers, if it is set to true - +// they are disabled and so tests will panic to avoid masking bugs. +var dontRecover = false + +type Queue struct { + name string + location string + hostname string + autogenMsgDomain string + wheel *TimeWheel + + dsnPipeline module.DeliveryTarget + + // Retry delay is calculated using the following formula: + // initialRetryTime * retryTimeScale ^ (TriesCount - 1) + + initialRetryTime time.Duration + retryTimeScale float64 + maxTries int + + // If any delivery is scheduled in less than postInitDelay + // after Init, its delay will be increased by postInitDelay. + // + // Say, if postInitDelay is 10 secs. + // Then if some message is scheduled to delivered 5 seconds + // after init, it will be actually delivered 15 seconds + // after start-up. + // + // This delay is added to make that if maddy is killed shortly + // after start-up for whatever reason it will not affect the queue. + postInitDelay time.Duration + + Log log.Logger + Target module.DeliveryTarget + + deliveryWg sync.WaitGroup + // Buffered channel used to restrict count of deliveries attempted + // in parallel. + deliverySemaphore chan struct{} +} + +type QueueMetadata struct { + MsgMeta *module.MsgMetadata + From string + + // Recipients that should be tried next. + // May or may not be equal to partialError.TemporaryFailed. + To []string + + // Information about previous failures. + // Preserved to be included in a bounce message. + FailedRcpts []string + TemporaryFailedRcpts []string + // All errors are converted to SMTPError we can serialize and + // also it is directly usable for bounce messages. + RcptErrs map[string]*smtp.SMTPError + + // Amount of times delivery *already tried*. + TriesCount map[string]int + + FirstAttempt time.Time + LastAttempt time.Time +} + +type queueSlot struct { + ID string + + // If nil - Hdr and Body are invalid, all values should be read from + // disk. + Meta *QueueMetadata + Hdr *textproto.Header + Body buffer.Buffer +} + +func NewQueue(_, instName string, _, inlineArgs []string) (module.Module, error) { + q := &Queue{ + name: instName, + initialRetryTime: 15 * time.Minute, + retryTimeScale: 1.25, + postInitDelay: 10 * time.Second, + Log: log.Logger{Name: "queue"}, + } + switch len(inlineArgs) { + case 0: + // Not inline definition. + case 1: + q.location = inlineArgs[0] + default: + return nil, errors.New("queue: wrong amount of inline arguments") + } + return q, nil +} + +func (q *Queue) Init(cfg *config.Map) error { + var maxParallelism int + cfg.Bool("debug", true, false, &q.Log.Debug) + cfg.Int("max_tries", false, false, 20, &q.maxTries) + cfg.Int("max_parallelism", false, false, 16, &maxParallelism) + cfg.String("location", false, false, q.location, &q.location) + cfg.Custom("target", false, true, nil, modconfig.DeliveryDirective, &q.Target) + cfg.String("hostname", true, true, "", &q.hostname) + cfg.String("autogenerated_msg_domain", true, false, "", &q.autogenMsgDomain) + cfg.Custom("bounce", false, false, nil, func(m *config.Map, node config.Node) (interface{}, error) { + return msgpipeline.New(m.Globals, node.Children) + }, &q.dsnPipeline) + if _, err := cfg.Process(); err != nil { + return err + } + + if q.dsnPipeline != nil { + if q.autogenMsgDomain == "" { + return errors.New("queue: autogenerated_msg_domain is required if bounce {} is specified") + } + + q.dsnPipeline.(*msgpipeline.MsgPipeline).Hostname = q.hostname + q.dsnPipeline.(*msgpipeline.MsgPipeline).Log = log.Logger{Name: "queue/pipeline", Debug: q.Log.Debug} + } + if q.location == "" && q.name == "" { + return errors.New("queue: need explicit location directive or inline argument if defined inline") + } + if q.location == "" { + q.location = filepath.Join(config.StateDirectory, q.name) + } + + // TODO: Check location write permissions. + if err := os.MkdirAll(q.location, os.ModePerm); err != nil { + return err + } + + return q.start(maxParallelism) +} + +func (q *Queue) start(maxParallelism int) error { + q.wheel = NewTimeWheel(q.dispatch) + q.deliverySemaphore = make(chan struct{}, maxParallelism) + + if err := q.readDiskQueue(); err != nil { + return err + } + + q.Log.Debugf("delivery target: %T", q.Target) + + return nil +} + +func (q *Queue) Close() error { + q.wheel.Close() + q.deliveryWg.Wait() + + return nil +} + +// discardBroken changes the name of metadata file to have .meta_broken +// extension. +// +// Further attempts to deliver (due to a timewheel) it will fail due to +// non-existent meta-data file. +// +// No error handling is done since this function is called from panic handler. +func (q *Queue) discardBroken(id string) { + err := os.Rename(filepath.Join(q.location, id+".meta"), filepath.Join(q.location, id+".meta_broken")) + if err != nil { + // Note: Global logger is used in case there is something wrong with Queue.Log. + log.Printf("can't mark the queue message as broken: %v", err) + } +} + +func (q *Queue) dispatch(value TimeSlot) { + slot := value.Value.(queueSlot) + + q.Log.Debugln("starting delivery for", slot.ID) + + q.deliveryWg.Add(1) + go func() { + q.Log.Debugln("waiting on delivery semaphore for", slot.ID) + q.deliverySemaphore <- struct{}{} + defer func() { + <-q.deliverySemaphore + q.deliveryWg.Done() + + if dontRecover { + return + } + + if err := recover(); err != nil { + stack := debug.Stack() + log.Printf("panic during queue dispatch %s: %v\n%s", slot.ID, err, stack) + q.discardBroken(slot.ID) + } + }() + + q.Log.Debugln("delivery semaphore acquired for", slot.ID) + var ( + meta *QueueMetadata + hdr textproto.Header + body buffer.Buffer + ) + if slot.Meta == nil { + var err error + meta, hdr, body, err = q.openMessage(slot.ID) + if err != nil { + q.Log.Error("read message", err, slot.ID) + return + } + if meta == nil { + panic("wtf") + } + } else { + meta = slot.Meta + hdr = *slot.Hdr + body = slot.Body + } + + q.tryDelivery(meta, hdr, body) + }() +} + +func toSMTPErr(err error) *smtp.SMTPError { + if err == nil { + return nil + } + + res := &smtp.SMTPError{ + Code: 554, + EnhancedCode: smtp.EnhancedCode{5, 0, 0}, + Message: "Internal server error", + } + + if exterrors.IsTemporaryOrUnspec(err) { + res.Code = 451 + res.EnhancedCode = smtp.EnhancedCode{4, 0, 0} + } + + ctxInfo := exterrors.Fields(err) + ctxCode, ok := ctxInfo["smtp_code"].(int) + if ok { + res.Code = ctxCode + } + ctxEnchCode, ok := ctxInfo["smtp_enchcode"].(smtp.EnhancedCode) + if ok { + res.EnhancedCode = ctxEnchCode + } + ctxMsg, ok := ctxInfo["smtp_msg"].(string) + if ok { + res.Message = ctxMsg + } + + if smtpErr, ok := err.(*smtp.SMTPError); ok { + log.Printf("plain SMTP error returned, this is deprecated") + res.Code = smtpErr.Code + res.EnhancedCode = smtpErr.EnhancedCode + res.Message = smtpErr.Message + } + + return res +} + +func (q *Queue) tryDelivery(meta *QueueMetadata, header textproto.Header, body buffer.Buffer) { + dl := target.DeliveryLogger(q.Log, meta.MsgMeta) + + partialErr := q.deliver(meta, header, body) + dl.Debugf("errors: %v", partialErr.Errs) + + // While iterating the list of recipients we also pick the smallest tries count + // and use it to calculate the delay for the next attempt. + smallestTriesCount := 999999 + + if meta.TriesCount == nil { + meta.TriesCount = make(map[string]int) + } + + // Check attempted recipients and corresponding errors. + // Split list into two parts: recipients that should be retried (newRcpts) + // and recipients DSN will be generated for. + newRcpts := make([]string, 0, len(partialErr.Errs)) + failedRcpts := make([]string, 0, len(partialErr.Errs)) + for _, rcpt := range meta.To { + rcptErr, ok := partialErr.Errs[rcpt] + if !ok { + dl.Msg("delivered", "rcpt", rcpt, "attempt", meta.TriesCount[rcpt]+1) + continue + } + + // Save last error (either temporary or permanent) for reporting in the DSN. + dl.Error("delivery attempt failed", rcptErr, "rcpt", rcpt) + meta.RcptErrs[rcpt] = toSMTPErr(rcptErr) + + temporary := exterrors.IsTemporaryOrUnspec(rcptErr) + if !temporary || meta.TriesCount[rcpt]+1 >= q.maxTries { + delete(meta.TriesCount, rcpt) + dl.Msg("not delivered, permanent error", "rcpt", rcpt) + failedRcpts = append(failedRcpts, rcpt) + continue + } + + // Temporary error, increase tries counter and requeue. + meta.TriesCount[rcpt]++ + newRcpts = append(newRcpts, rcpt) + + // See smallestTriesCount comment. + if count := meta.TriesCount[rcpt]; count < smallestTriesCount { + smallestTriesCount = count + } + } + + // Generate DSN for recipients that failed permanently this time. + if len(failedRcpts) != 0 { + q.emitDSN(meta, header, failedRcpts) + } + // No recipients to try, either all failed or all succeeded. + if len(newRcpts) == 0 { + q.removeFromDisk(meta.MsgMeta) + return + } + + meta.To = newRcpts + meta.LastAttempt = time.Now() + + if err := q.updateMetadataOnDisk(meta); err != nil { + dl.Error("meta-data update", err) + } + + nextTryTime := time.Now() + // Delay between retries grows exponentally, the formula is: + // initialRetryTime * retryTimeScale ^ (smallestTriesCount - 1) + dl.Debugf("delay: %v * %v ^ (%v - 1)", q.initialRetryTime, q.retryTimeScale, smallestTriesCount) + scaleFactor := time.Duration(math.Pow(q.retryTimeScale, float64(smallestTriesCount-1))) + nextTryTime = nextTryTime.Add(q.initialRetryTime * scaleFactor) + dl.Msg("will retry", + "attempts_count", meta.TriesCount, + "next_try_delay", time.Until(nextTryTime), + "rcpts", meta.To) + + q.wheel.Add(nextTryTime, queueSlot{ + ID: meta.MsgMeta.ID, + + // Do not keep (meta-)data in memory to reduce usage. At this point, + // it is safe on disk and next try will reread it. + Meta: nil, + Hdr: nil, + Body: nil, + }) +} + +func (q *Queue) deliver(meta *QueueMetadata, header textproto.Header, body buffer.Buffer) partialError { + dl := target.DeliveryLogger(q.Log, meta.MsgMeta) + perr := partialError{ + Errs: map[string]error{}, + statusLock: new(sync.Mutex), + } + + msgMeta := meta.MsgMeta.DeepCopy() + msgMeta.ID = msgMeta.ID + "-" + strconv.FormatInt(time.Now().Unix(), 16) + dl.Debugf("using message ID = %s", msgMeta.ID) + + msgCtx, msgTask := trace.NewTask(context.Background(), "Queue delivery") + defer msgTask.End() + + mailCtx, mailTask := trace.NewTask(msgCtx, "MAIL FROM") + delivery, err := q.Target.Start(mailCtx, msgMeta, meta.From) + mailTask.End() + if err != nil { + dl.Debugf("target.Start failed: %v", err) + for _, rcpt := range meta.To { + perr.Errs[rcpt] = err + } + return perr + } + dl.Debugf("target.Start OK") + + var acceptedRcpts []string + for _, rcpt := range meta.To { + rcptCtx, rcptTask := trace.NewTask(msgCtx, "RCPT TO") + if err := delivery.AddRcpt(rcptCtx, rcpt, smtp.RcptOptions{} /* TODO: DSN support */); err != nil { + dl.Debugf("delivery.AddRcpt %s failed: %v", rcpt, err) + perr.Errs[rcpt] = err + } else { + dl.Debugf("delivery.AddRcpt %s OK", rcpt) + acceptedRcpts = append(acceptedRcpts, rcpt) + } + rcptTask.End() + } + + if len(acceptedRcpts) == 0 { + dl.Debugf("delivery.Abort (no accepted recipients)") + if err := delivery.Abort(msgCtx); err != nil { + dl.Error("delivery.Abort failed", err) + } + return perr + } + + expandToPartialErr := func(err error) { + for _, rcpt := range acceptedRcpts { + perr.Errs[rcpt] = err + } + } + + bodyCtx, bodyTask := trace.NewTask(msgCtx, "DATA") + defer bodyTask.End() + + partDelivery, ok := delivery.(module.PartialDelivery) + if ok { + dl.Debugf("using delivery.BodyNonAtomic") + partDelivery.BodyNonAtomic(bodyCtx, &perr, header, body) + } else { + if err := delivery.Body(bodyCtx, header, body); err != nil { + dl.Debugf("delivery.Body failed: %v", err) + expandToPartialErr(err) + } + dl.Debugf("delivery.Body OK") + } + + allFailed := true + for _, rcpt := range acceptedRcpts { + if perr.Errs[rcpt] == nil { + allFailed = false + } + } + if allFailed { + // No recipients succeeded. + dl.Debugf("delivery.Abort (all recipients failed)") + if err := delivery.Abort(bodyCtx); err != nil { + dl.Msg("delivery.Abort failed", err) + } + return perr + } + + if err := delivery.Commit(bodyCtx); err != nil { + dl.Debugf("delivery.Commit failed: %v", err) + expandToPartialErr(err) + } + dl.Debugf("delivery.Commit OK") + + return perr +} + +type queueDelivery struct { + q *Queue + meta *QueueMetadata + + header textproto.Header + body buffer.Buffer +} + +func (qd *queueDelivery) AddRcpt(ctx context.Context, rcptTo string, _ smtp.RcptOptions) error { + qd.meta.To = append(qd.meta.To, rcptTo) + return nil +} + +func (qd *queueDelivery) Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error { + defer trace.StartRegion(ctx, "queue/Body").End() + + // Body buffer initially passed to us may not be valid after "delivery" to queue completes. + // storeNewMessage returns a new buffer object created from message blob stored on disk. + storedBody, err := qd.q.storeNewMessage(qd.meta, header, body) + if err != nil { + return err + } + + qd.body = storedBody + qd.header = header + return nil +} + +func (qd *queueDelivery) Abort(ctx context.Context) error { + defer trace.StartRegion(ctx, "queue/Abort").End() + + if qd.body != nil { + qd.q.removeFromDisk(qd.meta.MsgMeta) + } + return nil +} + +func (qd *queueDelivery) Commit(ctx context.Context) error { + defer trace.StartRegion(ctx, "queue/Commit").End() + + if qd.meta == nil { + panic("queue: double Commit") + } + + qd.q.wheel.Add(time.Time{}, queueSlot{ + ID: qd.meta.MsgMeta.ID, + Meta: qd.meta, + Hdr: &qd.header, + Body: qd.body, + }) + qd.meta = nil + qd.body = nil + return nil +} + +func (q *Queue) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + meta := &QueueMetadata{ + MsgMeta: msgMeta, + From: mailFrom, + RcptErrs: map[string]*smtp.SMTPError{}, + FirstAttempt: time.Now(), + LastAttempt: time.Now(), + } + return &queueDelivery{q: q, meta: meta}, nil +} + +func (q *Queue) removeFromDisk(msgMeta *module.MsgMetadata) { + id := msgMeta.ID + dl := target.DeliveryLogger(q.Log, msgMeta) + + // Order is important. + // If we remove header and body but can't remove meta now - readDiskQueue + // will detect and report it. + headerPath := filepath.Join(q.location, id+".header") + if err := os.Remove(headerPath); err != nil { + dl.Error("failed to remove header from disk", err) + } + bodyPath := filepath.Join(q.location, id+".body") + if err := os.Remove(bodyPath); err != nil { + dl.Error("failed to remove body from disk", err) + } + metaPath := filepath.Join(q.location, id+".meta") + if err := os.Remove(metaPath); err != nil { + dl.Error("failed to remove meta-data from disk", err) + } + dl.Debugf("removed message from disk") +} + +func (q *Queue) readDiskQueue() error { + dirInfo, err := os.ReadDir(q.location) + if err != nil { + return err + } + + // TODO(GH #209): Rewrite this function to pass all sub-tests in TestQueueDelivery_DeserializationCleanUp/NoMeta. + + loadedCount := 0 + for _, entry := range dirInfo { + // We start loading from meta-data files and then check whether ID.header and ID.body exist. + // This allows us to properly detect dangling body files. + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".meta") { + continue + } + id := entry.Name()[:len(entry.Name())-5] + + meta, err := q.readMessageMeta(id) + if err != nil { + q.Log.Printf("failed to read meta-data, skipping: %v (msg ID = %s)", err, id) + continue + } + + // Check header file existence. + if _, err := os.Stat(filepath.Join(q.location, id+".header")); err != nil { + if os.IsNotExist(err) { + q.Log.Printf("header file doesn't exist for msg ID = %s", id) + q.tryRemoveDanglingFile(id + ".meta") + q.tryRemoveDanglingFile(id + ".body") + } else { + q.Log.Printf("skipping nonstat'able header file: %v (msg ID = %s)", err, id) + } + continue + } + + // Check body file existence. + if _, err := os.Stat(filepath.Join(q.location, id+".body")); err != nil { + if os.IsNotExist(err) { + q.Log.Printf("body file doesn't exist for msg ID = %s", id) + q.tryRemoveDanglingFile(id + ".meta") + q.tryRemoveDanglingFile(id + ".header") + } else { + q.Log.Printf("skipping nonstat'able body file: %v (msg ID = %s)", err, id) + } + continue + } + + smallestTriesCount := 999999 + for _, count := range meta.TriesCount { + if smallestTriesCount > count { + smallestTriesCount = count + } + } + nextTryTime := meta.LastAttempt + scaleFactor := time.Duration(math.Pow(q.retryTimeScale, float64(smallestTriesCount-1))) + nextTryTime = nextTryTime.Add(q.initialRetryTime * scaleFactor) + + if time.Until(nextTryTime) < q.postInitDelay { + nextTryTime = time.Now().Add(q.postInitDelay) + } + + q.Log.Debugf("will try to deliver (msg ID = %s) in %v (%v)", id, time.Until(nextTryTime), nextTryTime) + q.wheel.Add(nextTryTime, queueSlot{ + ID: id, + }) + loadedCount++ + } + + if loadedCount != 0 { + q.Log.Printf("loaded %d saved queue entries", loadedCount) + } + + return nil +} + +func (q *Queue) storeNewMessage(meta *QueueMetadata, header textproto.Header, body buffer.Buffer) (buffer.Buffer, error) { + id := meta.MsgMeta.ID + + headerPath := filepath.Join(q.location, id+".header") + headerFile, err := os.Create(headerPath) + if err != nil { + return nil, err + } + defer headerFile.Close() + + if err := textproto.WriteHeader(headerFile, header); err != nil { + q.tryRemoveDanglingFile(id + ".header") + return nil, err + } + + bodyReader, err := body.Open() + if err != nil { + q.tryRemoveDanglingFile(id + ".header") + return nil, err + } + defer bodyReader.Close() + + bodyPath := filepath.Join(q.location, id+".body") + bodyFile, err := os.Create(bodyPath) + if err != nil { + return nil, err + } + defer bodyFile.Close() + + if _, err := io.Copy(bodyFile, bodyReader); err != nil { + q.tryRemoveDanglingFile(id + ".body") + q.tryRemoveDanglingFile(id + ".header") + return nil, err + } + + if err := q.updateMetadataOnDisk(meta); err != nil { + q.tryRemoveDanglingFile(id + ".body") + q.tryRemoveDanglingFile(id + ".header") + return nil, err + } + + if err := headerFile.Sync(); err != nil { + return nil, err + } + + if err := bodyFile.Sync(); err != nil { + return nil, err + } + + return buffer.FileBuffer{Path: bodyPath, LenHint: body.Len()}, nil +} + +func (q *Queue) updateMetadataOnDisk(meta *QueueMetadata) error { + metaPath := filepath.Join(q.location, meta.MsgMeta.ID+".meta") + + var file *os.File + var err error + if runtime.GOOS == "windows" { + file, err = os.Create(metaPath) + if err != nil { + return err + } + } else { + file, err = os.Create(metaPath + ".new") + if err != nil { + return err + } + } + defer file.Close() + + metaCopy := *meta + metaCopy.MsgMeta = meta.MsgMeta.DeepCopy() + metaCopy.MsgMeta.Conn = nil + + if err := json.NewEncoder(file).Encode(metaCopy); err != nil { + return err + } + + if err := file.Sync(); err != nil { + return err + } + + if runtime.GOOS != "windows" { + if err := os.Rename(metaPath+".new", metaPath); err != nil { + return err + } + } + + return nil +} + +func (q *Queue) readMessageMeta(id string) (*QueueMetadata, error) { + metaPath := filepath.Join(q.location, id+".meta") + file, err := os.Open(metaPath) + if err != nil { + return nil, err + } + defer file.Close() + + meta := &QueueMetadata{} + + meta.MsgMeta = &module.MsgMetadata{} + + // There is a couple of problems we have to solve before we would be able to + // serialize ConnState. + // 1. future.Future can't be serialized. + // 2. net.Addr can't be deserialized because we don't know the concrete type. + + if err := json.NewDecoder(file).Decode(meta); err != nil { + return nil, err + } + + return meta, nil +} + +type BufferedReadCloser struct { + *bufio.Reader + io.Closer +} + +func (q *Queue) tryRemoveDanglingFile(name string) { + if err := os.Remove(filepath.Join(q.location, name)); err != nil { + q.Log.Error("dangling file remove failed", err) + return + } + q.Log.Printf("removed dangling file %s", name) +} + +func (q *Queue) openMessage(id string) (*QueueMetadata, textproto.Header, buffer.Buffer, error) { + meta, err := q.readMessageMeta(id) + if err != nil { + return nil, textproto.Header{}, nil, err + } + + bodyPath := filepath.Join(q.location, id+".body") + _, err = os.Stat(bodyPath) + if err != nil { + if os.IsNotExist(err) { + q.tryRemoveDanglingFile(id + ".meta") + } + return nil, textproto.Header{}, nil, err + } + body := buffer.FileBuffer{Path: bodyPath} + + headerPath := filepath.Join(q.location, id+".header") + headerFile, err := os.Open(headerPath) + if err != nil { + if os.IsNotExist(err) { + q.tryRemoveDanglingFile(id + ".meta") + q.tryRemoveDanglingFile(id + ".body") + } + return nil, textproto.Header{}, nil, err + } + + bufferedHeader := bufio.NewReader(headerFile) + header, err := textproto.ReadHeader(bufferedHeader) + if err != nil { + return nil, textproto.Header{}, nil, err + } + + return meta, header, body, nil +} + +func (q *Queue) InstanceName() string { + return q.name +} + +func (q *Queue) Name() string { + return "queue" +} + +func (q *Queue) emitDSN(meta *QueueMetadata, header textproto.Header, failedRcpts []string) { + // If, apparently, we have no DSN msgpipeline configured - do nothing. + if q.dsnPipeline == nil { + return + } + + // Null return-path, used in DSNs. + if meta.MsgMeta.OriginalFrom == "" { + return + } + + dsnID, err := module.GenerateMsgID() + if err != nil { + q.Log.Error("rand.Rand error", err) + return + } + + dsnEnvelope := dsn.Envelope{ + MsgID: "<" + dsnID + "@" + q.autogenMsgDomain + ">", + From: "MAILER-DAEMON@" + q.autogenMsgDomain, + To: meta.MsgMeta.OriginalFrom, + } + mtaInfo := dsn.ReportingMTAInfo{ + ReportingMTA: q.hostname, + XSender: meta.From, + XMessageID: meta.MsgMeta.ID, + ArrivalDate: meta.FirstAttempt, + LastAttemptDate: meta.LastAttempt, + } + if !meta.MsgMeta.DontTraceSender && meta.MsgMeta.Conn != nil { + mtaInfo.ReceivedFromMTA = meta.MsgMeta.Conn.Hostname + } + + rcptInfo := make([]dsn.RecipientInfo, 0, len(meta.RcptErrs)) + for _, rcpt := range failedRcpts { + rcptErr := meta.RcptErrs[rcpt] + // rcptErr is stored in RcptErrs using the effective recipient address, + // not the original one. + + originalRcpt := meta.MsgMeta.OriginalRcpts[rcpt] + if originalRcpt != "" { + rcpt = originalRcpt + } + + rcptInfo = append(rcptInfo, dsn.RecipientInfo{ + FinalRecipient: rcpt, + Action: dsn.ActionFailed, + Status: rcptErr.EnhancedCode, + DiagnosticCode: rcptErr, + }) + } + + var dsnBodyBlob bytes.Buffer + dl := target.DeliveryLogger(q.Log, meta.MsgMeta) + dsnHeader, err := dsn.GenerateDSN(meta.MsgMeta.SMTPOpts.UTF8, dsnEnvelope, mtaInfo, rcptInfo, header, &dsnBodyBlob) + if err != nil { + dl.Error("failed to generate fail DSN", err) + return + } + dsnBody := buffer.MemoryBuffer{Slice: dsnBodyBlob.Bytes()} + + dsnMeta := &module.MsgMetadata{ + ID: dsnID, + SMTPOpts: smtp.MailOptions{ + UTF8: meta.MsgMeta.SMTPOpts.UTF8, + RequireTLS: meta.MsgMeta.SMTPOpts.RequireTLS, + }, + } + dl.Msg("generated failed DSN", "dsn_id", dsnID) + + msgCtx, msgTask := trace.NewTask(context.Background(), "DSN Delivery") + defer msgTask.End() + + mailCtx, mailTask := trace.NewTask(msgCtx, "MAIL FROM") + dsnDelivery, err := q.dsnPipeline.Start(mailCtx, dsnMeta, "") + mailTask.End() + if err != nil { + dl.Error("failed to enqueue DSN", err, "dsn_id", dsnID) + return + } + + defer func() { + if err != nil { + dl.Error("failed to enqueue DSN", err, "dsn_id", dsnID) + if err := dsnDelivery.Abort(msgCtx); err != nil { + dl.Error("failed to abort DSN delivery", err, "dsn_id", dsnID) + } + } + }() + + rcptCtx, rcptTask := trace.NewTask(msgCtx, "RCPT TO") + if err = dsnDelivery.AddRcpt(rcptCtx, meta.From, smtp.RcptOptions{}); err != nil { + rcptTask.End() + return + } + rcptTask.End() + + bodyCtx, bodyTask := trace.NewTask(msgCtx, "DATA") + if err = dsnDelivery.Body(bodyCtx, dsnHeader, dsnBody); err != nil { + bodyTask.End() + return + } + if err = dsnDelivery.Commit(bodyCtx); err != nil { + bodyTask.End() + return + } + bodyTask.End() +} + +func init() { + module.Register("target.queue", NewQueue) +} diff --git a/internal/target/queue/queue_test.go b/internal/target/queue/queue_test.go new file mode 100644 index 0000000..ff9a4f6 --- /dev/null +++ b/internal/target/queue/queue_test.go @@ -0,0 +1,828 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package queue + +import ( + "bytes" + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +// newTestQueue returns properly initialized Queue object usable for testing. +// +// See newTestQueueDir to create testing queue from an existing directory. +// It is called responsibility to remove queue directory created by this function. +func newTestQueue(t *testing.T, target module.DeliveryTarget) *Queue { + return newTestQueueDir(t, target, t.TempDir()) +} + +func cleanQueue(t *testing.T, q *Queue) { + t.Log("--- queue.Close") + if err := q.Close(); err != nil { + t.Fatal("queue.Close:", err) + } +} + +func newTestQueueDir(t *testing.T, target module.DeliveryTarget, dir string) *Queue { + mod, _ := NewQueue("", "queue", nil, nil) + q := mod.(*Queue) + q.initialRetryTime = 0 + q.retryTimeScale = 1 + q.postInitDelay = 0 + q.maxTries = 5 + q.location = dir + q.Target = target + + if testing.Verbose() { + q.Log = testutils.Logger(t, "queue") + } else { + q.Log = log.Logger{Out: log.NopOutput{}} + } + + if err := q.start(1); err != nil { + panic(err) + } + + return q +} + +// unreliableTarget is a module.DeliveryTarget implementation that stores +// messages to a slice and sometimes fails with the specified error. +type unreliableTarget struct { + committed chan testutils.Msg + aborted chan testutils.Msg + + // Amount of completed deliveries (both failed and succeeded) + passedMessages int + + // To make unreliableTarget fail Commit for N-th delivery, set N-1-th + // element of this slice to wanted error object. If slice is + // nil/empty or N is bigger than its size - delivery will succeed. + bodyFailures []error + bodyFailuresPartial []map[string]error + rcptFailures []map[string]error +} + +type unreliableTargetDelivery struct { + ut *unreliableTarget + msg testutils.Msg +} + +type unreliableTargetDeliveryPartial struct { + *unreliableTargetDelivery +} + +func (utd *unreliableTargetDelivery) AddRcpt(ctx context.Context, rcptTo string, _ smtp.RcptOptions) error { + if len(utd.ut.rcptFailures) > utd.ut.passedMessages { + rcptErrs := utd.ut.rcptFailures[utd.ut.passedMessages] + if err := rcptErrs[rcptTo]; err != nil { + return err + } + } + + utd.msg.RcptTo = append(utd.msg.RcptTo, rcptTo) + return nil +} + +func (utd *unreliableTargetDelivery) Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error { + if utd.ut.bodyFailuresPartial != nil { + return errors.New("partial failure occurred, no additional information available") + } + + r, _ := body.Open() + utd.msg.Body, _ = io.ReadAll(r) + + if len(utd.ut.bodyFailures) > utd.ut.passedMessages { + return utd.ut.bodyFailures[utd.ut.passedMessages] + } + + return nil +} + +func (utd *unreliableTargetDeliveryPartial) BodyNonAtomic(ctx context.Context, c module.StatusCollector, header textproto.Header, body buffer.Buffer) { + r, _ := body.Open() + utd.msg.Body, _ = io.ReadAll(r) + + if len(utd.ut.bodyFailuresPartial) > utd.ut.passedMessages { + for rcpt, err := range utd.ut.bodyFailuresPartial[utd.ut.passedMessages] { + c.SetStatus(rcpt, err) + } + } +} + +func (utd *unreliableTargetDelivery) Abort(ctx context.Context) error { + utd.ut.passedMessages++ + if utd.ut.aborted != nil { + utd.ut.aborted <- utd.msg + } + return nil +} + +func (utd *unreliableTargetDelivery) Commit(ctx context.Context) error { + utd.ut.passedMessages++ + if utd.ut.committed != nil { + utd.ut.committed <- utd.msg + } + return nil +} + +func (ut *unreliableTarget) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + if ut.bodyFailuresPartial != nil { + return &unreliableTargetDeliveryPartial{ + &unreliableTargetDelivery{ + ut: ut, + msg: testutils.Msg{ + MsgMeta: msgMeta, + MailFrom: mailFrom, + }, + }, + }, nil + } + return &unreliableTargetDelivery{ + ut: ut, + msg: testutils.Msg{ + MsgMeta: msgMeta, + MailFrom: mailFrom, + }, + }, nil +} + +func readMsgChanTimeout(t *testing.T, ch <-chan testutils.Msg, timeout time.Duration) *testutils.Msg { + t.Helper() + timer := time.NewTimer(timeout) + select { + case msg := <-ch: + return &msg + case <-timer.C: + t.Fatal("chan read timed out") + return nil + } +} + +func checkQueueDir(t *testing.T, q *Queue, expectedIDs []string) { + t.Helper() + // We use the map to lookups and also to mark messages we found + // we can report missing entries. + expectedMap := make(map[string]bool, len(expectedIDs)) + for _, id := range expectedIDs { + expectedMap[id] = false + } + + dir, err := os.ReadDir(q.location) + if err != nil { + t.Fatalf("failed to read queue directory: %v", err) + } + + // Queue implementation uses file names in the following format: + // DELIVERY_ID.SOMETHING + for _, file := range dir { + if file.IsDir() { + t.Fatalf("queue should not create subdirectories in the store, but there is %s dir in it", file.Name()) + } + + nameParts := strings.Split(file.Name(), ".") + if len(nameParts) != 2 { + t.Fatalf("did the queue files name format changed? got %s", file.Name()) + } + + _, ok := expectedMap[nameParts[0]] + if !ok { + t.Errorf("message with unexpected Msg ID %s is stored in queue store", nameParts[0]) + continue + } + + expectedMap[nameParts[0]] = true + } + + for id, found := range expectedMap { + if !found { + t.Errorf("expected message with Msg ID %s is missing from queue store", id) + } + } +} + +func TestQueueDelivery(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{committed: make(chan testutils.Msg, 10)} + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // Wait for the delivery to complete and stop processing. + msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) + q.Close() + + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}, "") + + // There should be no queued messages. + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_PermanentFail_NonPartial(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + bodyFailures: []error{ + exterrors.WithTemporary(errors.New("you shall not pass"), false), + }, + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // Queue will abort a delivery if it fails for all recipients. + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + q.Close() + + // Delivery is failed permanently, hence no retry should be rescheduled. + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_PermanentFail_Partial(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + bodyFailuresPartial: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("you shall not pass"), false), + "tester2@example.org": exterrors.WithTemporary(errors.New("you shall not pass"), false), + }, + }, + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // This this is similar to the previous test, but checks PartialDelivery processing logic. + // Here delivery fails for recipients too, but this is reported using PartialDelivery. + + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + q.Close() + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_TemporaryFail(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + bodyFailures: []error{ + exterrors.WithTemporary(errors.New("you shall not pass"), true), + }, + aborted: make(chan testutils.Msg, 10), + committed: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // Delivery should be aborted, because it failed for all recipients. + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + + // Second retry, should work fine. + msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}, "") + + q.Close() + // No more retries scheduled, queue storage is clear. + defer checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_TemporaryFail_Partial(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + bodyFailuresPartial: []map[string]error{ + { + "tester2@example.org": exterrors.WithTemporary(errors.New("go away"), true), + }, + }, + aborted: make(chan testutils.Msg, 10), + committed: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // Committed, tester1@example.org - ok. + msg := readMsgChanTimeout(t, dt.committed, 5000*time.Second) + // Side note: unreliableTarget adds recipients to the msg object even if they were rejected + // later using a partial error. So slice below is all recipients that were submitted by + // the queue. + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}, "") + + // committed #2, tester2@example.org - ok + msg = readMsgChanTimeout(t, dt.committed, 5000*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester2@example.org"}, "") + + q.Close() + // No more retries scheduled, queue storage is clear. + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_MultipleAttempts(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + bodyFailuresPartial: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("you shall not pass 1"), false), + "tester2@example.org": exterrors.WithTemporary(errors.New("you shall not pass 2"), true), + }, + { + "tester2@example.org": exterrors.WithTemporary(errors.New("you shall not pass 3"), true), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org", "tester3@example.org"}) + + // Committed because delivery to tester3@example.org is succeeded. + msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) + // Side note: This slice contains all recipients submitted by the queue, even if + // they were rejected later using partialError. + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org", "tester2@example.org", "tester3@example.org"}, "") + + // tester1 is failed permanently, should not be retried. + // tester2 is failed temporary, should be retried. + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + + // Third attempt... tester2 delivered. + msg = readMsgChanTimeout(t, dt.committed, 5*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester2@example.org"}, "") + + q.Close() + // No more retries should be scheduled. + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_PermanentRcptReject(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), false), + }, + }, + committed: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.org", []string{"tester1@example.org", "tester2@example.org"}) + + // Committed, tester2@example.org succeeded. + msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.org", []string{"tester2@example.org"}, "") + + q.Close() + // No more retries should be scheduled. + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_TemporaryRcptReject(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), true), + }, + }, + committed: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + // First attempt: + // tester1 - temp. fail + // tester2 - ok + // Second attempt: + // tester1 - ok + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) + // Unlike previous tests where unreliableTarget rejected recipients by partialError, here they are rejected + // by AddRcpt directly, so they are NOT saved by the target. + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester2@example.org"}, "") + + msg = readMsgChanTimeout(t, dt.committed, 5*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org"}, "") + + q.Close() + // No more retries should be scheduled. + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_SerializationRoundtrip(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), true), + }, + }, + committed: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + // This is the most tricky test because it is racy and I have no idea what can be done to avoid it. + // It relies on us calling Close before queue msgpipeline decides to retry delivery. + // Hence retry delay is increased from 0ms used in other tests to make it reliable. + q.initialRetryTime = 1 * time.Second + + // To make sure we will not time out due to post-init delay. + q.postInitDelay = 0 + + deliveryID := testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // Standard partial delivery, retry will be scheduled for tester1@example.org. + msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester2@example.org"}, "") + + // Then stop it. + q.Close() + + // Make sure it is saved. + checkQueueDir(t, q, []string{deliveryID}) + + // Then reinit it. + q = newTestQueueDir(t, &dt, q.location) + + // Wait for retry and check it. + msg = readMsgChanTimeout(t, dt.committed, 5*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester1@example.org"}, "") + + // Close it again. + q.Close() + // No more retries should be scheduled. + checkQueueDir(t, q, []string{}) +} + +func TestQueueDelivery_DeserlizationCleanUp(t *testing.T) { + t.Parallel() + + test := func(t *testing.T, fileSuffix string) { + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), true), + }, + }, + committed: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + // This is the most tricky test because it is racy and I have no idea what can be done to avoid it. + // It relies on us calling Close before queue msgpipeline decides to retry delivery. + // Hence retry delay is increased from 0ms used in other tests to make it reliable. + q.initialRetryTime = 1 * time.Second + + // To make sure we will not time out due to post-init delay. + q.postInitDelay = 0 + + deliveryID := testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // Standard partial delivery, retry will be scheduled for tester1@example.org. + msg := readMsgChanTimeout(t, dt.committed, 5*time.Second) + testutils.CheckMsgID(t, msg, "tester@example.com", []string{"tester2@example.org"}, "") + + q.Close() + + if err := os.Remove(filepath.Join(q.location, deliveryID+fileSuffix)); err != nil { + t.Fatal(err) + } + + // Dangling files should be removed during load. + q = newTestQueueDir(t, &dt, q.location) + q.Close() + + // Nothing should be left. + checkQueueDir(t, q, []string{}) + } + + t.Run("NoMeta", func(t *testing.T) { + t.Skip("Not implemented") + test(t, ".meta") + }) + t.Run("NoBody", func(t *testing.T) { + test(t, ".body") + }) + t.Run("NoHeader", func(t *testing.T) { + test(t, ".header") + }) +} + +func TestQueueDelivery_AbortIfNoRecipients(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), true), + "tester2@example.org": exterrors.WithTemporary(errors.New("go away"), true), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + readMsgChanTimeout(t, dt.aborted, 5*time.Second) +} + +func TestQueueDelivery_AbortNoDangling(t *testing.T) { + t.Parallel() + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), true), + "tester2@example.org": exterrors.WithTemporary(errors.New("go away"), true), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + defer cleanQueue(t, q) + + // Copied from testutils.DoTestDelivery. + IDRaw := sha1.Sum([]byte(t.Name())) + encodedID := hex.EncodeToString(IDRaw[:]) + + body := buffer.MemoryBuffer{Slice: []byte("foobar\r\n")} + ctx := module.MsgMetadata{ + DontTraceSender: true, + ID: encodedID, + } + delivery, err := q.Start(context.Background(), &ctx, "test3@example.org") + if err != nil { + t.Fatalf("unexpected Start err: %v", err) + } + for _, rcpt := range [...]string{"test@example.org", "test2@example.org"} { + if err := delivery.AddRcpt(context.Background(), rcpt, smtp.RcptOptions{}); err != nil { + t.Fatalf("unexpected AddRcpt err for %s: %v", rcpt, err) + } + } + if err := delivery.Body(context.Background(), textproto.Header{}, body); err != nil { + t.Fatalf("unexpected Body err: %v", err) + } + if err := delivery.Abort(context.Background()); err != nil { + t.Fatalf("unexpected Abort err: %v", err) + } + + checkQueueDir(t, q, []string{}) +} + +func TestQueueDSN(t *testing.T) { + t.Parallel() + + dsnTarget := unreliableTarget{ + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), false), + "tester2@example.org": exterrors.WithTemporary(errors.New("go away"), false), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + q.hostname = "mx.example.org" + q.autogenMsgDomain = "example.org" + q.dsnPipeline = &dsnTarget + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.com", []string{"tester1@example.org", "tester2@example.org"}) + + // Wait for message delivery attempt to complete (aborted because all recipients fail). + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + + // Wait for DSN. + msg := readMsgChanTimeout(t, dsnTarget.committed, 5*time.Second) + + if msg.MailFrom != "" { + t.Fatalf("wrong MAIL FROM address in DSN: %v", msg.MailFrom) + } + if !reflect.DeepEqual(msg.RcptTo, []string{"tester@example.com"}) { + t.Fatalf("wrong RCPT TO address in DSN: %v", msg.RcptTo) + } +} + +func TestQueueDSN_FromEmptyAddr(t *testing.T) { + t.Parallel() + + dsnTarget := unreliableTarget{ + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), false), + "tester2@example.org": exterrors.WithTemporary(errors.New("go away"), false), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + q.hostname = "mx.example.org" + q.autogenMsgDomain = "example.org" + q.dsnPipeline = &dsnTarget + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "", []string{"tester1@example.org", "tester2@example.org"}) + + // Wait for message delivery attempt to complete (aborted because all recipients fail). + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + + time.Sleep(1 * time.Second) + + // There should be no DSN for it. + if dsnTarget.passedMessages != 0 { + t.Errorf("dsnTarget accepted %d messages", dsnTarget.passedMessages) + } + checkQueueDir(t, q, []string{}) +} + +func TestQueueDSN_NoDSNforDSN(t *testing.T) { + t.Parallel() + + dsnTarget := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester@example.org": exterrors.WithTemporary(errors.New("go away"), false), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "tester1@example.org": exterrors.WithTemporary(errors.New("go away"), false), + "tester2@example.org": exterrors.WithTemporary(errors.New("go away"), false), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + q.hostname = "mx.example.org" + q.autogenMsgDomain = "example.org" + q.dsnPipeline = &dsnTarget + defer cleanQueue(t, q) + + testutils.DoTestDelivery(t, q, "tester@example.org", []string{"tester1@example.org", "tester2@example.org"}) + + // Wait for message delivery attempt to complete (aborted because all recipients fail). + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + + // DSN will be emitted but will fail, so 'aborted' + readMsgChanTimeout(t, dsnTarget.aborted, 5*time.Second) + + time.Sleep(1 * time.Second) + + // There should be no DSN for DSN (dsnTarget handled one message - the DSN itself). + if dsnTarget.passedMessages != 1 { + t.Errorf("dsnTarget accepted %d messages", dsnTarget.passedMessages) + } + checkQueueDir(t, q, []string{}) +} + +func TestQueueDSN_RcptRewrite(t *testing.T) { + t.Parallel() + + dsnTarget := unreliableTarget{ + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + + dt := unreliableTarget{ + rcptFailures: []map[string]error{ + { + "test@example.org": exterrors.WithTemporary(errors.New("go away"), false), + "test2@example.org": exterrors.WithTemporary(errors.New("go away"), false), + }, + }, + committed: make(chan testutils.Msg, 10), + aborted: make(chan testutils.Msg, 10), + } + q := newTestQueue(t, &dt) + q.hostname = "mx.example.org" + q.autogenMsgDomain = "example.org" + q.dsnPipeline = &dsnTarget + defer cleanQueue(t, q) + + IDRaw := sha1.Sum([]byte(t.Name())) + encodedID := hex.EncodeToString(IDRaw[:]) + + body := buffer.MemoryBuffer{Slice: []byte("foobar\r\n")} + ctx := module.MsgMetadata{ + DontTraceSender: true, + OriginalFrom: "test3@example.org", + OriginalRcpts: map[string]string{ + "test@example.org": "test+public@example.com", + "test2@example.org": "test2+public@example.com", + }, + ID: encodedID, + } + delivery, err := q.Start(context.Background(), &ctx, "test3@example.org") + if err != nil { + t.Fatalf("unexpected Start err: %v", err) + } + for _, rcpt := range [...]string{"test@example.org", "test2@example.org"} { + if err := delivery.AddRcpt(context.Background(), rcpt, smtp.RcptOptions{}); err != nil { + t.Fatalf("unexpected AddRcpt err for %s: %v", rcpt, err) + } + } + if err := delivery.Body(context.Background(), textproto.Header{}, body); err != nil { + t.Fatalf("unexpected Body err: %v", err) + } + if err := delivery.Commit(context.Background()); err != nil { + t.Fatalf("unexpected Commit err: %v", err) + } + + // Wait for message delivery attempt to complete (aborted because all recipients fail). + readMsgChanTimeout(t, dt.aborted, 5*time.Second) + + // Wait for DSN. + msg := readMsgChanTimeout(t, dsnTarget.committed, 5*time.Second) + + if msg.MailFrom != "" { + t.Fatalf("wrong MAIL FROM address in DSN: %v", msg.MailFrom) + } + if !reflect.DeepEqual(msg.RcptTo, []string{"test3@example.org"}) { + t.Fatalf("wrong RCPT TO address in DSN: %v", msg.RcptTo) + } + + if bytes.Contains(msg.Body, []byte("test@example.org")) || bytes.Contains(msg.Body, []byte("test2@example.org")) { + t.Errorf("DSN contents mention real final addresses") + } + if !bytes.Contains(msg.Body, []byte("test+public@example.com")) || !bytes.Contains(msg.Body, []byte("test2+public@example.com")) { + t.Errorf("DSN contents do not mention original addresses") + } +} + +func init() { + dontRecover = true +} diff --git a/internal/target/queue/timewheel.go b/internal/target/queue/timewheel.go new file mode 100644 index 0000000..060804b --- /dev/null +++ b/internal/target/queue/timewheel.go @@ -0,0 +1,146 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package queue + +import ( + "container/list" + "sync" + "sync/atomic" + "time" +) + +type TimeSlot struct { + Time time.Time + Value interface{} +} + +type TimeWheel struct { + stopped uint32 + + slots *list.List + slotsLock sync.Mutex + + updateNotify chan time.Time + stopNotify chan struct{} + + dispatch func(TimeSlot) +} + +func NewTimeWheel(dispatch func(TimeSlot)) *TimeWheel { + tw := &TimeWheel{ + slots: list.New(), + stopNotify: make(chan struct{}), + updateNotify: make(chan time.Time), + dispatch: dispatch, + } + go tw.tick() + return tw +} + +func (tw *TimeWheel) Add(target time.Time, value interface{}) { + if atomic.LoadUint32(&tw.stopped) == 1 { + // Already stopped, ignore. + return + } + + if value == nil { + panic("can't insert nil objects into TimeWheel queue") + } + + tw.slotsLock.Lock() + tw.slots.PushBack(TimeSlot{Time: target, Value: value}) + tw.slotsLock.Unlock() + + tw.updateNotify <- target +} + +func (tw *TimeWheel) Close() { + atomic.StoreUint32(&tw.stopped, 1) + + // Idempotent Close is convenient sometimes. + if tw.stopNotify == nil { + return + } + + tw.stopNotify <- struct{}{} + <-tw.stopNotify + + tw.stopNotify = nil + + close(tw.updateNotify) +} + +func (tw *TimeWheel) tick() { + for { + now := time.Now() + // Look for list element closest to now. + tw.slotsLock.Lock() + var closestSlot TimeSlot + var closestEl *list.Element + for e := tw.slots.Front(); e != nil; e = e.Next() { + slot := e.Value.(TimeSlot) + if slot.Time.Sub(now) < closestSlot.Time.Sub(now) || closestSlot.Value == nil { + closestSlot = slot + closestEl = e + } + } + tw.slotsLock.Unlock() + // Only this goroutine removes elements from TimeWheel so we can be safe using closestSlot. + + // Queue is empty. Just wait until update. + if closestEl == nil { + select { + case <-tw.updateNotify: + continue + case <-tw.stopNotify: + tw.stopNotify <- struct{}{} + return + } + } + + timer := time.NewTimer(closestSlot.Time.Sub(now)) + + selectloop: + for { + select { + case <-timer.C: + tw.slotsLock.Lock() + tw.slots.Remove(closestEl) + tw.slotsLock.Unlock() + + tw.dispatch(closestSlot) + + break selectloop + case newTarget := <-tw.updateNotify: + // Avoid unnecessary restarts if new target is not going to affect our + // current wait time. + if closestSlot.Time.Sub(now) <= newTarget.Sub(now) { + continue + } + + timer.Stop() + // Recalculate new slot time. + break selectloop + case <-tw.stopNotify: + tw.stopNotify <- struct{}{} + return + } + } + } +} diff --git a/internal/target/queue/timewheel_test.go b/internal/target/queue/timewheel_test.go new file mode 100644 index 0000000..d758b60 --- /dev/null +++ b/internal/target/queue/timewheel_test.go @@ -0,0 +1,127 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package queue + +import ( + "testing" + "time" +) + +func TestTimeWheelAdd(t *testing.T) { + t.Parallel() + + called := make(chan TimeSlot) + + w := NewTimeWheel(func(slot TimeSlot) { + called <- slot + }) + defer w.Close() + + w.Add(time.Now().Add(1*time.Second), 1) + + slot := <-called + if val, _ := slot.Value.(int); val != 1 { + t.Errorf("Wrong slot value: %v", slot.Value) + } +} + +func TestTimeWheelAdd_Ordering(t *testing.T) { + t.Parallel() + + called := make(chan TimeSlot) + + w := NewTimeWheel(func(slot TimeSlot) { + called <- slot + }) + defer w.Close() + + w.Add(time.Now().Add(1*time.Second), 1) + w.Add(time.Now().Add(1250*time.Millisecond), 2) + + slot := <-called + if val, _ := slot.Value.(int); val != 1 { + t.Errorf("Wrong first slot value: %v", slot.Value) + } + slot = <-called + if val, _ := slot.Value.(int); val != 2 { + t.Errorf("Wrong second slot value: %v", slot.Value) + } +} + +func TestTimeWheelAdd_Restart(t *testing.T) { + t.Parallel() + + called := make(chan TimeSlot) + + w := NewTimeWheel(func(slot TimeSlot) { + called <- slot + }) + defer w.Close() + + w.Add(time.Now().Add(1*time.Second), 1) + w.Add(time.Now().Add(500*time.Millisecond), 2) + + slot := <-called + if val, _ := slot.Value.(int); val != 2 { + t.Errorf("Wrong first slot value: %v", slot.Value) + } + slot = <-called + if val, _ := slot.Value.(int); val != 1 { + t.Errorf("Wrong second slot value: %v", slot.Value) + } +} + +func TestTimeWheelAdd_MissingGotoBug(t *testing.T) { + t.Parallel() + + called := make(chan TimeSlot) + + w := NewTimeWheel(func(slot TimeSlot) { + called <- slot + }) + defer w.Close() + + w.Add(time.Now().Add(90000*time.Hour), 1) // practically newer + w.Add(time.Now().Add(500*time.Millisecond), 2) // should correctly restart + + slot := <-called + if val, _ := slot.Value.(int); val != 2 { + t.Errorf("Wrong first slot value: %v", slot.Value) + } +} + +func TestTimeWheelAdd_EmptyUpdWait(t *testing.T) { + t.Parallel() + + called := make(chan TimeSlot) + + w := NewTimeWheel(func(slot TimeSlot) { + called <- slot + }) + defer w.Close() + + time.Sleep(500 * time.Millisecond) + + w.Add(time.Now().Add(1*time.Second), 1) + + slot := <-called + if val, _ := slot.Value.(int); val != 1 { + t.Errorf("Wrong slot value: %v", slot.Value) + } +} diff --git a/internal/target/received.go b/internal/target/received.go new file mode 100644 index 0000000..051e5a6 --- /dev/null +++ b/internal/target/received.go @@ -0,0 +1,107 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package target + +import ( + "context" + "errors" + "net" + "strings" + "time" + + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/module" +) + +func SanitizeForHeader(raw string) string { + return strings.Replace(raw, "\n", "", -1) +} + +func GenerateReceived(ctx context.Context, msgMeta *module.MsgMetadata, ourHostname, mailFrom string) (string, error) { + if msgMeta.Conn == nil { + return "", errors.New("can't generate Received for a locally generated message") + } + + builder := strings.Builder{} + + // Empirically guessed value that should be enough to fit + // the entire value in most cases. + builder.Grow(256 + len(msgMeta.Conn.Hostname)) + + if !msgMeta.DontTraceSender && (strings.Contains(msgMeta.Conn.Proto, "SMTP") || + strings.Contains(msgMeta.Conn.Proto, "LMTP")) { + // INTERNATIONALIZATION: See RFC 6531 Section 3.7.3. + hostname, err := dns.SelectIDNA(msgMeta.SMTPOpts.UTF8, msgMeta.Conn.Hostname) + if err == nil { + builder.WriteString("from ") + builder.WriteString(hostname) + } + + if tcpAddr, ok := msgMeta.Conn.RemoteAddr.(*net.TCPAddr); ok { + builder.WriteString(" (") + if msgMeta.Conn.RDNSName != nil { + rdnsName, err := msgMeta.Conn.RDNSName.GetContext(ctx) + if err == nil && rdnsName != nil && rdnsName.(string) != "" { + // INTERNATIONALIZATION: See RFC 6531 Section 3.7.3. + encoded, err := dns.SelectIDNA(msgMeta.SMTPOpts.UTF8, rdnsName.(string)) + if err == nil { + builder.WriteString(encoded) + builder.WriteRune(' ') + } + } + } + builder.WriteRune('[') + builder.WriteString(tcpAddr.IP.String()) + builder.WriteString("])") + } + } + + if ourHostname != "" { + ourHostname, err := dns.SelectIDNA(msgMeta.SMTPOpts.UTF8, ourHostname) + if err == nil { + builder.WriteString(" by ") + builder.WriteString(SanitizeForHeader(ourHostname)) + } + } + + if mailFrom != "" { + // INTERNATIONALIZATION: See RFC 6531 Section 3.7.3. + mailFrom, err := address.SelectIDNA(msgMeta.SMTPOpts.UTF8, mailFrom) + if err == nil { + builder.WriteString(" (envelope-sender <") + builder.WriteString(SanitizeForHeader(mailFrom)) + builder.WriteString(">)") + } + } + + if msgMeta.Conn.Proto != "" { + builder.WriteString(" with ") + if msgMeta.SMTPOpts.UTF8 { + builder.WriteString("UTF8") + } + builder.WriteString(msgMeta.Conn.Proto) + } + builder.WriteString(" id ") + builder.WriteString(msgMeta.ID) + builder.WriteString("; ") + builder.WriteString(time.Now().Format(time.RFC1123Z)) + + return strings.TrimSpace(builder.String()), nil +} diff --git a/internal/target/remote/connect.go b/internal/target/remote/connect.go new file mode 100644 index 0000000..f9d317e --- /dev/null +++ b/internal/target/remote/connect.go @@ -0,0 +1,390 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "context" + "crypto/tls" + "errors" + "net" + "runtime/trace" + "sort" + "time" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/smtpconn" +) + +type mxConn struct { + *smtpconn.C + + // Domain this MX belongs to. + domain string + dnssecOk bool + + // Errors occurred previously on this connection. + errored bool + + reuseLimit int + + // Amount of times connection was used for an SMTP transaction. + transactions int + lastUseAt time.Time + + // MX/TLS security level established for this connection. + mxLevel module.MXLevel + tlsLevel module.TLSLevel +} + +func (c *mxConn) Usable() bool { + if c.C == nil || c.transactions > c.reuseLimit || c.C.Client() == nil || c.errored { + return false + } + return c.C.Client().Reset() == nil +} + +func (c *mxConn) LastUseAt() time.Time { + return c.lastUseAt +} + +func (c *mxConn) Close() error { + return c.C.Close() +} + +func isVerifyError(err error) bool { + var e *tls.CertificateVerificationError + return errors.As(err, &e) +} + +// connect attempts to connect to the MX, first trying STARTTLS with X.509 +// verification but falling back to unauthenticated TLS or plaintext as +// necessary. +// +// Return values: +// - tlsLevel TLS security level that was estabilished. +// - tlsErr Error that prevented TLS from working if tlsLevel != TLSAuthenticated +func (rd *remoteDelivery) connect(ctx context.Context, conn mxConn, host string, tlsCfg *tls.Config) (tlsLevel module.TLSLevel, tlsErr, err error) { + tlsLevel = module.TLSAuthenticated + if rd.rt.tlsConfig != nil { + tlsCfg = rd.rt.tlsConfig.Clone() + tlsCfg.ServerName = host + } + + rd.Log.DebugMsg("trying", "remote_server", host, "domain", conn.domain) + +retry: + // smtpconn.C default TLS behavior is not useful for us, we want to handle + // TLS errors separately hence starttls=false. + _, err = conn.Connect(ctx, config.Endpoint{ + Host: host, + Port: smtpPort, + }, false, nil) + if err != nil { + return module.TLSNone, nil, err + } + + starttlsOk, _ := conn.Client().Extension("STARTTLS") + if starttlsOk && tlsCfg != nil { + if err := conn.Client().StartTLS(tlsCfg); err != nil { + // Here we just issue STARTTLS command. If it fails for some + // reason - this is either a connection problem or server actively + // rejecting STARTTLS (despite advertising STARTTLS). + // We err on the caution side here and do not perform any fallbacks. + conn.DirectClose() + return module.TLSNone, nil, err + } + + // TLS handshake is deferred to here, this is where we check errors and allow fallback. + if err := conn.Client().Hello(rd.rt.hostname); err != nil { + tlsErr = err + + // Attempt TLS without authentication. It is still better than + // plaintext and we might be able to actually authenticate the + // server using DANE-EE/DANE-TA later. + // + // Check tlsLevel is to avoid looping forever if the same verify + // error happens with InsecureSkipVerify too (e.g. certificate is + // *too* broken). + if isVerifyError(err) && tlsLevel == module.TLSAuthenticated { + rd.Log.Error("TLS verify error, trying without authentication", err, "remote_server", host, "domain", conn.domain) + tlsCfg.InsecureSkipVerify = true + tlsLevel = module.TLSEncrypted + + // TODO: Check go-smtp code to make TLS verification errors + // non-sticky so we can properly send QUIT in this case. + conn.DirectClose() + + goto retry + } + + rd.Log.Error("TLS error, trying plaintext", err, "remote_server", host, "domain", conn.domain) + tlsCfg = nil + tlsLevel = module.TLSNone + conn.DirectClose() + + goto retry + } + } else { + tlsLevel = module.TLSNone + } + + return tlsLevel, tlsErr, nil +} + +func (rd *remoteDelivery) attemptMX(ctx context.Context, conn *mxConn, record *net.MX) error { + mxLevel := module.MXNone + + connCtx, cancel := context.WithCancel(ctx) + // Cancel async policy lookups if rd.connect fails. + defer cancel() + + for _, p := range rd.policies { + policyLevel, err := p.CheckMX(connCtx, mxLevel, conn.domain, record.Host, conn.dnssecOk) + if err != nil { + return err + } + if policyLevel > mxLevel { + mxLevel = policyLevel + } + + p.PrepareConn(ctx, record.Host) + } + + tlsLevel, tlsErr, err := rd.connect(connCtx, *conn, record.Host, rd.rt.tlsConfig) + if err != nil { + return err + } + + // Make decision based on the policy and connection state. + // + // Note: All policy errors are marked as temporary to give the local admin + // chance to troubleshoot them without losing messages. + + tlsState, _ := conn.Client().TLSConnectionState() + for _, p := range rd.policies { + policyLevel, err := p.CheckConn(connCtx, mxLevel, tlsLevel, conn.domain, record.Host, tlsState) + if err != nil { + conn.Close() + return exterrors.WithFields(err, map[string]interface{}{"tls_err": tlsErr}) + } + if policyLevel > tlsLevel { + tlsLevel = policyLevel + } + } + + conn.mxLevel = mxLevel + conn.tlsLevel = tlsLevel + + mxLevelCnt.WithLabelValues(rd.rt.Name(), mxLevel.String()).Inc() + tlsLevelCnt.WithLabelValues(rd.rt.Name(), tlsLevel.String()).Inc() + + return nil +} + +func (rd *remoteDelivery) connectionForDomain(ctx context.Context, domain string) (*mxConn, error) { + if c, ok := rd.connections[domain]; ok { + return c, nil + } + + pooledConn, err := rd.rt.pool.Get(ctx, domain) + if err != nil { + return nil, err + } + + var conn *mxConn + // Ignore pool for connections with REQUIRETLS to avoid "pool poisoning" + // where attacker can make messages indeliverable by forcing reuse of old + // connection with weaker security. + if pooledConn != nil && !rd.msgMeta.SMTPOpts.RequireTLS { + conn = pooledConn.(*mxConn) + rd.Log.Msg("reusing cached connection", "domain", domain, "transactions_counter", conn.transactions, + "local_addr", conn.LocalAddr(), "remote_addr", conn.RemoteAddr()) + } else { + rd.Log.DebugMsg("opening new connection", "domain", domain, "cache_ignored", pooledConn != nil) + conn, err = rd.newConn(ctx, domain) + if err != nil { + return nil, err + } + } + + if rd.msgMeta.SMTPOpts.RequireTLS { + if conn.tlsLevel < module.TLSAuthenticated { + conn.Close() + return nil, &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 30}, + Message: "TLS it not available or unauthenticated but required (REQUIRETLS)", + Misc: map[string]interface{}{ + "tls_level": conn.tlsLevel, + }, + } + } + if conn.mxLevel < module.MX_MTASTS { + conn.Close() + return nil, &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 30}, + Message: "Failed to establish the MX record authenticity (REQUIRETLS)", + Misc: map[string]interface{}{ + "mx_level": conn.mxLevel, + }, + } + } + } + + region := trace.StartRegion(ctx, "remote/limits.TakeDest") + if err := rd.rt.limits.TakeDest(ctx, domain); err != nil { + region.End() + conn.Close() + return nil, err + } + region.End() + + // Relaxed REQUIRETLS mode is not conforming to the specification strictly + // but allows to start deploying client support for REQUIRETLS without the + // requirement for servers in the whole world to support it. The assumption + // behind it is that MX for the recipient domain is the final destination + // and all other forwarders behind it already have secure connection to + // each other. Therefore it is enough to enforce strict security only on + // the path to the MX even if it does not support the REQUIRETLS to propagate + // this requirement further. + if ok, _ := conn.Client().Extension("REQUIRETLS"); rd.rt.relaxedREQUIRETLS && !ok { + rd.msgMeta.SMTPOpts.RequireTLS = false + } + + if err := conn.Mail(ctx, rd.mailFrom, rd.msgMeta.SMTPOpts); err != nil { + conn.Close() + return nil, err + } + conn.lastUseAt = time.Now() + + rd.connections[domain] = conn + return conn, nil +} + +func (rd *remoteDelivery) newConn(ctx context.Context, domain string) (*mxConn, error) { + conn := mxConn{ + reuseLimit: rd.rt.connReuseLimit, + C: smtpconn.New(), + domain: domain, + lastUseAt: time.Now(), + } + + conn.Dialer = rd.rt.dialer + conn.Log = rd.Log + conn.Hostname = rd.rt.hostname + conn.AddrInSMTPMsg = true + if rd.rt.connectTimeout != 0 { + conn.ConnectTimeout = rd.rt.connectTimeout + } + if rd.rt.commandTimeout != 0 { + conn.CommandTimeout = rd.rt.commandTimeout + } + if rd.rt.submissionTimeout != 0 { + conn.SubmissionTimeout = rd.rt.submissionTimeout + } + + for _, p := range rd.policies { + p.PrepareDomain(ctx, domain) + } + + region := trace.StartRegion(ctx, "remote/LookupMX") + dnssecOk, records, err := rd.lookupMX(ctx, domain) + region.End() + if err != nil { + return nil, err + } + conn.dnssecOk = dnssecOk + + var lastErr error + region = trace.StartRegion(ctx, "remote/Connect+TLS") + for _, record := range records { + if record.Host == "." { + return nil, &exterrors.SMTPError{ + Code: 556, + EnhancedCode: exterrors.EnhancedCode{5, 1, 10}, + Message: "Domain does not accept email (null MX)", + } + } + + if err := rd.attemptMX(ctx, &conn, record); err != nil { + if len(records) != 0 { + rd.Log.Error("cannot use MX", err, "remote_server", record.Host, "domain", domain) + } + lastErr = err + continue + } + break + } + region.End() + + // Still not connected? Bail out. + if conn.Client() == nil { + return nil, &exterrors.SMTPError{ + Code: exterrors.SMTPCode(lastErr, 451, 550), + EnhancedCode: exterrors.SMTPEnchCode(lastErr, exterrors.EnhancedCode{0, 4, 0}), + Message: "No usable MXs, last err: " + lastErr.Error(), + TargetName: "remote", + Err: lastErr, + Misc: map[string]interface{}{ + "domain": domain, + }, + } + } + + return &conn, nil +} + +func (rd *remoteDelivery) lookupMX(ctx context.Context, domain string) (dnssecOk bool, records []*net.MX, err error) { + if rd.rt.extResolver != nil { + dnssecOk, records, err = rd.rt.extResolver.AuthLookupMX(context.Background(), domain) + } else { + records, err = rd.rt.resolver.LookupMX(ctx, dns.FQDN(domain)) + } + if err != nil { + reason, misc := exterrors.UnwrapDNSErr(err) + return false, nil, &exterrors.SMTPError{ + Code: exterrors.SMTPCode(err, 451, 554), + EnhancedCode: exterrors.SMTPEnchCode(err, exterrors.EnhancedCode{0, 4, 4}), + Message: "MX lookup error", + TargetName: "remote", + Reason: reason, + Err: err, + Misc: misc, + } + } + + sort.Slice(records, func(i, j int) bool { + return records[i].Pref < records[j].Pref + }) + + // Fallback to A/AAA RR when no MX records are present as + // required by RFC 5321 Section 5.1. + if len(records) == 0 { + records = append(records, &net.MX{ + Host: domain, + Pref: 0, + }) + } + + return dnssecOk, records, err +} diff --git a/internal/target/remote/dane.go b/internal/target/remote/dane.go new file mode 100644 index 0000000..2e7dd77 --- /dev/null +++ b/internal/target/remote/dane.go @@ -0,0 +1,158 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "crypto/tls" + "crypto/x509" + "time" + + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" +) + +// Used to override verification time for DANE-TA tests. +var verifyDANETime time.Time + +// verifyDANE checks whether TLSA records require TLS use and match the +// certificate and name used by the server. +// +// overridePKIX result indicates whether DANE should make server authentication +// succeed even if PKIX/X.509 verification fails. That is, if InsecureSkipVerify +// is used and verifyDANE returns overridePKIX=true, the server certificate +// should trusted. +func verifyDANE(recs []dns.TLSA, connState tls.ConnectionState) (overridePKIX bool, err error) { + tlsErr := &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 1}, + Message: "TLS is required but unsupported or failed (enforced by DANE)", + TargetName: "remote", + Misc: map[string]interface{}{ + "remote_server": connState.ServerName, + }, + } + + // See https://tools.ietf.org/html/rfc7672#section-2.2 for requirements of + // TLS discovery. + // We assume upstream resolver will generate an error if the DNSSEC + // signature is bogus so this case is "DNSSEC-authenticated denial of existence". + if len(recs) == 0 { + return false, nil + } + + // Require TLS even if all records are not usable, per Section 2.2 of RFC 7672. + if !connState.HandshakeComplete { + return false, tlsErr + } + + // Ignore invalid records. + var ( + eeRecs []dns.TLSA + taRecs []dns.TLSA + ) + for _, rec := range recs { + switch rec.MatchingType { + case 0, 1, 2: + default: + continue + } + switch rec.Selector { + case 0, 1: + default: + continue + } + + switch rec.Usage { + case 2: + taRecs = append(taRecs, rec) + case 3: + eeRecs = append(eeRecs, rec) + default: + continue + } + } + + // Authentication is not required if all records are unusable, see + // RFC 7672 Section 2.1.1. + if len(eeRecs) == 0 && len(taRecs) == 0 { + return false, nil + } + + for _, rec := range eeRecs { + if rec.Verify(connState.PeerCertificates[0]) == nil { + // https://tools.ietf.org/html/rfc7672#section-3.1.1 + // - SAN/CN are not considered. + // - Expired certificates are fine too. + return true, nil + } + } + + // Don't bother building a temporary certificate pool if there are no + // records to check. + if len(taRecs) == 0 { + return true, &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "No matching TLSA records", + TargetName: "remote", + Misc: map[string]interface{}{ + "remote_server": connState.ServerName, + }, + } + } + + // Collect certificates presented by the server as possible intermediates. + // Add all certificates from the chain that match any record to the root + // pool. + opts := x509.VerifyOptions{ + DNSName: connState.ServerName, + Intermediates: x509.NewCertPool(), + Roots: x509.NewCertPool(), + CurrentTime: verifyDANETime, + } + for _, cert := range connState.PeerCertificates { + root := false + for _, rec := range taRecs { + if cert.IsCA && rec.Verify(cert) == nil { + opts.Roots.AddCert(cert) + root = true + } + } + if !root { + opts.Intermediates.AddCert(cert) + } + } + + // ... then run the standard X.509 verification. This will verify that the + // server certificate chains to any of asserted TA certificates. + if _, err := connState.PeerCertificates[0].Verify(opts); err == nil { + return true, nil + } + + // There are valid records, but none matched. + return false, &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "No matching TLSA records", + TargetName: "remote", + Misc: map[string]interface{}{ + "remote_server": connState.ServerName, + }, + } +} diff --git a/internal/target/remote/dane_delivery_test.go b/internal/target/remote/dane_delivery_test.go new file mode 100644 index 0000000..2b921c7 --- /dev/null +++ b/internal/target/remote/dane_delivery_test.go @@ -0,0 +1,477 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "crypto/tls" + "net" + "strconv" + "testing" + + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" + miekgdns "github.com/miekg/dns" +) + +func targetWithExtResolver(t *testing.T, zones map[string]mockdns.Zone) (*mockdns.Server, *Target) { + dnsSrv, err := mockdns.NewServerWithLogger(zones, testutils.Logger(t, "mockdns"), false) + if err != nil { + t.Fatal(err) + } + + dialer := net.Dialer{} + dialer.Resolver = &net.Resolver{} + dnsSrv.PatchNet(dialer.Resolver) + addr := dnsSrv.LocalAddr().(*net.UDPAddr) + + extResolver, err := dns.NewExtResolver() + if err != nil { + t.Fatal(err) + } + extResolver.Cfg.Servers = []string{addr.IP.String()} + extResolver.Cfg.Port = strconv.Itoa(addr.Port) + + tgt := testTarget(t, zones, extResolver, []module.MXAuthPolicy{ + testDANEPolicy(t, extResolver), + }) + return dnsSrv, tgt +} + +func tlsaRecord(name string, usage, matchType, selector uint8, cert string) map[miekgdns.Type][]miekgdns.RR { + return map[miekgdns.Type][]miekgdns.RR{ + miekgdns.Type(miekgdns.TypeTLSA): { + &miekgdns.TLSA{ + Hdr: miekgdns.RR_Header{ + Name: name, + Class: miekgdns.ClassINET, + Rrtype: miekgdns.TypeTLSA, + Ttl: 9999, + }, + Usage: usage, + MatchingType: matchType, + Selector: selector, + Certificate: cert, + }, + }, + } +} + +func TestRemoteDelivery_DANE_Ok(t *testing.T) { + _, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + // RFC 7672, Section 2.2.2. "Non-CNAME" case. + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.example.invalid.": { + AD: true, + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cf"), + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.policies = append(tgt.policies, + &localPolicy{ + minTLSLevel: module.TLSAuthenticated, // Established via DANE instead of PKIX. + }, + ) + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_DANE_CNAMEd_1(t *testing.T) { + _, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + // RFC 7672, Section 2.2.2. "Secure CNAME" case - TLSA at CNAME matches. + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + CNAME: "mx.cname.invalid.", + }, + "mx.cname.invalid.": { + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.cname.invalid.": { + AD: true, + Misc: tlsaRecord( + "_25._tcp.mx.cname.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cf"), + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.policies = append(tgt.policies, + &localPolicy{ + minTLSLevel: module.TLSAuthenticated, // Established via DANE instead of PKIX. + }, + ) + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_DANE_CNAMEd_2(t *testing.T) { + _, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + // RFC 7672, Section 2.2.2. "Secure CNAME" case - TLSA at initial name matches. + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + CNAME: "mx.cname.invalid.", + }, + "_25._tcp.mx.example.invalid.": { + AD: true, + Misc: tlsaRecord( + "_25._tcp.mx.cname.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cf"), + }, + "mx.cname.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.policies = append(tgt.policies, + &localPolicy{ + minTLSLevel: module.TLSAuthenticated, // Established via DANE instead of PKIX. + }, + ) + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_DANE_InsecureCNAMEDest(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + // RFC 7672, Section 2.2.2. "Insecure CNAME" case - initial name is secure. + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + CNAME: "mx.cname.invalid.", + }, + "_25._tcp.mx.example.invalid.": { + AD: true, + // This is the record that activates DANE but does not match the cert + // => delivery is failed. + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cb"), + }, + "_25._tcp.mx.cname.invalid.": { + AD: false, + // This is the record that matches the cert and would make delivery succeed + // but it should not be considered since AD=false. + Misc: tlsaRecord( + "_25._tcp.mx.cname.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cf"), + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.tlsConfig = clientCfg + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued but should not") + } +} + +func TestRemoteDelivery_DANE_NonAD_TLSA_Ignore(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + // RFC 7672, Section 2.2.2. "Non-CNAME" case - initial name is insecure. + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.example.invalid.": { + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cb"), + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_DANE_NonADIgnore_CNAME(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + // RFC 7672, Section 2.2.2. "Insecure CNAME" case - initial name is insecure. + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + CNAME: "mx.cname.invalid.", + }, + "mx.cname.invalid.": { + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.cname.invalid.": { + AD: true, + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cb"), + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_DANE_SkipAUnauth(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.example.invalid.": { + AD: false, + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "invalid hex will cause serialization error and no response will be sent"), + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.tlsConfig = clientCfg + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_DANE_Mismatch(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.example.invalid.": { + AD: true, + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "ffb5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cf"), + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.tlsConfig = clientCfg + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued but should not") + } +} + +func TestRemoteDelivery_DANE_NoRecord(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.tlsConfig = clientCfg + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_DANE_LookupErr(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.example.invalid.": { + Err: &net.DNSError{}, + }, + } + + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + tgt.tlsConfig = clientCfg + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued but should not") + } +} + +func TestRemoteDelivery_DANE_NoTLS(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.example.invalid.": { + AD: true, + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cf"), + }, + } + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued but should not") + } +} + +func TestRemoteDelivery_DANE_TLSError(t *testing.T) { + _, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + AD: true, + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + "_25._tcp.mx.example.invalid.": { + AD: true, + Misc: tlsaRecord( + "_25._tcp.mx.example.invalid.", + 3, 1, 1, "a9b5cb4d02f996f6385debe9a8952f1af1f4aec7eae0f37c2cd6d0d8ee8391cf"), + }, + } + dnsSrv, tgt := targetWithExtResolver(t, zones) + defer dnsSrv.Close() + + // Cause failure through version incompatibility. + tgt.tlsConfig = &tls.Config{ + MaxVersion: tls.VersionTLS12, + MinVersion: tls.VersionTLS12, + } + srv.TLSConfig.MinVersion = tls.VersionTLS11 + srv.TLSConfig.MaxVersion = tls.VersionTLS11 + + // DANE should prevent the fallback to plaintext. + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued but should not") + } +} diff --git a/internal/target/remote/dane_test.go b/internal/target/remote/dane_test.go new file mode 100644 index 0000000..470fbbb --- /dev/null +++ b/internal/target/remote/dane_test.go @@ -0,0 +1,227 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/pem" + "testing" + "time" + + "github.com/miekg/dns" +) + +// These certificates are related like this: +// +// Root A -> Intermediate A -> Leaf A +// Root B -> LeafB +var ( + rootA = `-----BEGIN CERTIFICATE----- +MIIBMDCB46ADAgECAhRDwag3n5CG90BEO87zEMAPejn6YTAFBgMrZXAwFjEUMBIG +A1UEAxMLVGVzdCBSb290IEEwHhcNMjAxMTI4MjExODA4WhcNMzAxMTI2MjExODA4 +WjAWMRQwEgYDVQQDEwtUZXN0IFJvb3QgQTAqMAUGAytlcAMhADXMzcRec5ocluNR +ExnnNT7I5fmcpjf2P4ik5k0DJNbco0MwQTAPBgNVHRMBAf8EBTADAQH/MA8GA1Ud +DwEB/wQFAwMHBAAwHQYDVR0OBBYEFM5b/b1di1vA+YpMZcsF4K7N1LbaMAUGAytl +cANBAAZ0XTxBDjN9VGPqWjXrYqGPUqbjm4JD3PeHUB4YGH+MNTgeVIlU8qCLIXtM +9kmAkCk7+j5G8p0gMjJMNygeuwE= +-----END CERTIFICATE-----` + intermediateA = `-----BEGIN CERTIFICATE----- +MIIBWjCCAQygAwIBAgIUEOd619/8HC1pWXxaEpQ1vUZOe7wwBQYDK2VwMBYxFDAS +BgNVBAMTC1Rlc3QgUm9vdCBBMB4XDTIwMTEyODIxMTk0M1oXDTMwMTEyNjIxMTk0 +M1owHjEcMBoGA1UEAxMTVGVzdCBJbnRlcm1lZGlhdGUgQTAqMAUGAytlcAMhAFgW +aZz5316olEIHn1Q4RTPd2u/EjN2bo+Cn3EmSlFxto2QwYjAPBgNVHRMBAf8EBTAD +AQH/MA8GA1UdDwEB/wQFAwMHBAAwHQYDVR0OBBYEFB0P00Qphygy+KgkI9tjihFD +ELxhMB8GA1UdIwQYMBaAFM5b/b1di1vA+YpMZcsF4K7N1LbaMAUGAytlcANBAJJH +zsS8ahEjdyRCNUlsPalZiKW8N3G0LnwdVKFhVfcCT+RTRcrMP7vjuWsbJyD5e7hu +z2eCI68xreLQlNySdQ0= +-----END CERTIFICATE-----` + leafA = `-----BEGIN CERTIFICATE----- +MIIBjzCCAUGgAwIBAgIUONvbCs6r9zKFM3IAPRMdrNiJpNgwBQYDK2VwMB4xHDAa +BgNVBAMTE1Rlc3QgSW50ZXJtZWRpYXRlIEEwHhcNMjAxMTI4MjEyMTIyWhcNMzAx +MTI2MjEyMTIyWjAWMRQwEgYDVQQDEwtUZXN0IExlYWYgQTAqMAUGAytlcAMhABIj +W7gwY78RCWHs9eSIdy4x4MXjzdhZwgNSNHHCp5pAo4GYMIGVMAwGA1UdEwEB/wQC +MAAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBUGA1UdEQQOMAyCCm1h +ZGR5LnRlc3QwDwYDVR0PAQH/BAUDAweAADAdBgNVHQ4EFgQU9PFQCnG5fNpNPXUT +8rCuylS6tVwwHwYDVR0jBBgwFoAUHQ/TRCmHKDL4qCQj22OKEUMQvGEwBQYDK2Vw +A0EAGdvHA4VLxpUeUu1Vjom2YX3MukPJG0a3/dB3HiAWWpxMgWfU+Ftie7noaNcI +oUW+M8my46dqN6oXSHU47/QjDg== +-----END CERTIFICATE-----` + rootB = `-----BEGIN CERTIFICATE----- +MIIBMDCB46ADAgECAhRXD7xuPkipDyxyCtm8pZaxhuulaDAFBgMrZXAwFjEUMBIG +A1UEAxMLVGVzdCBSb290IEIwHhcNMjAxMTI4MjExODMwWhcNMzAxMTI2MjExODMw +WjAWMRQwEgYDVQQDEwtUZXN0IFJvb3QgQjAqMAUGAytlcAMhAPOIGJJh5jK8N/Vc +lLrFpysV+SiZjT1Cmt7hoFtMrlbTo0MwQTAPBgNVHRMBAf8EBTADAQH/MA8GA1Ud +DwEB/wQFAwMHBAAwHQYDVR0OBBYEFOLGYf4mkhKbZPwZKCv952tfz/KDMAUGAytl +cANBAOX2gb6ud8CAvOsCgw6uaRm0+jMDVZfkAkNuCIO6cJ/WYfdvuXYXu3e88SuI +gri++h118PomIzJ5PHAaCYsFPgQ= +-----END CERTIFICATE-----` + leafB = `-----BEGIN CERTIFICATE----- +MIIBhzCCATmgAwIBAgIUR2bVQ/Cu4j7Td5TdbWd6Q0LEpOgwBQYDK2VwMBYxFDAS +BgNVBAMTC1Rlc3QgUm9vdCBCMB4XDTIwMTEyODIxMjE0M1oXDTMwMTEyNjIxMjE0 +M1owFjEUMBIGA1UEAxMLVGVzdCBMZWFmIEIwKjAFBgMrZXADIQBiHCTUxF3UxPIV +M/o5OkTtmUrI7AInOvMa0dchU4iJXqOBmDCBlTAMBgNVHRMBAf8EAjAAMB0GA1Ud +JQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAVBgNVHREEDjAMggptYWRkeS50ZXN0 +MA8GA1UdDwEB/wQFAwMHgAAwHQYDVR0OBBYEFPYZPubaAXyr6kXs3khqpMNfdHKK +MB8GA1UdIwQYMBaAFOLGYf4mkhKbZPwZKCv952tfz/KDMAUGAytlcANBABlOwVxE +h7vYmaMYoyOSF1GQiB0ZLsGUjrTNHDnv0+Xp8xG5Td5mGnBi/4Ehq39PdLrj2T7j +3Xy0aiqdDomvwQY= +-----END CERTIFICATE-----` +) + +func parsePEMCert(blob string) *x509.Certificate { + block, _ := pem.Decode([]byte(blob)) + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + panic(err) + } + return cert +} + +func singleTlsaRecord(usage, matchType, selector uint8, cert string) dns.TLSA { + return dns.TLSA{ + Hdr: dns.RR_Header{ + Name: "maddy.test.", + Class: dns.ClassINET, + Rrtype: dns.TypeTLSA, + Ttl: 9999, + }, + Usage: usage, + MatchingType: matchType, + Selector: selector, + Certificate: cert, + } +} + +func keySHA256(blob string) string { + cert := parsePEMCert(blob) + hash := sha256.Sum256(cert.RawSubjectPublicKeyInfo) + return hex.EncodeToString(hash[:]) +} + +func TestVerifyDANE(t *testing.T) { + verifyDANETime = time.Unix(1606600100, 0) + test := func(name string, recs []dns.TLSA, connState tls.ConnectionState, expectErr bool) { + t.Helper() + t.Run(name, func(t *testing.T) { + t.Helper() + _, err := verifyDANE(recs, connState) + if (err != nil) != expectErr { + t.Error("err:", err, "expectErr:", expectErr) + } + }) + } + + // RFC 7672, Section 2.2: + // An "insecure" TLSA RRset or DNSSEC-authenticated denial of existence + // of the TLSA records: + // A connection to the MTA SHOULD be made using (pre-DANE) + // opportunistic TLS; + // + // "Insecure" TLSA RRset results in verifyDANE not being called at all, + // but for the latter (authenticated denial of existence) it is still + // called and should be tested for. + // + // More specific tests for TLSA RRset discovery (including CNAME + // shenanigans) are in dane_delivery_test.go. + test("no TLSA, TLS", []dns.TLSA{}, tls.ConnectionState{ + HandshakeComplete: true, + }, false) + test("no TLSA, no TLS", []dns.TLSA{}, tls.ConnectionState{ + HandshakeComplete: false, + }, false) + + // RFC 7272, Section 2.2: + // A "secure" non-empty TLSA RRset where all the records are unusable: + // Any connection to the MTA MUST be made via TLS, but authentication + // is not required. + test("unusable TLSA, TLS", []dns.TLSA{ + singleTlsaRecord(4, 1, 2, "whatever"), + singleTlsaRecord(4, 5, 2, "whatever"), + singleTlsaRecord(4, 1, 1, "whatever"), + }, tls.ConnectionState{ + HandshakeComplete: true, + PeerCertificates: []*x509.Certificate{parsePEMCert(leafA)}, + }, false) + test("unusable TLSA, no TLS", []dns.TLSA{ + singleTlsaRecord(4, 1, 2, "whatever"), + }, tls.ConnectionState{ + HandshakeComplete: false, + }, true) + + // RFC 7672, Section 2.2: + // A "secure" TLSA RRset with at least one usable record: Any + // connection to the MTA MUST employ TLS encryption and MUST + // authenticate the SMTP server using the techniques discussed in the + // rest of this document. + test("DANE-EE, non-self-signed", []dns.TLSA{ + singleTlsaRecord(3, 1, 1, keySHA256(leafA)), + }, tls.ConnectionState{ + HandshakeComplete: true, + PeerCertificates: []*x509.Certificate{parsePEMCert(leafA)}, + }, false) + test("DANE-EE, multiple records", []dns.TLSA{ + singleTlsaRecord(3, 1, 1, keySHA256(leafB)), + singleTlsaRecord(3, 1, 1, keySHA256(leafA)), + }, tls.ConnectionState{ + HandshakeComplete: true, + PeerCertificates: []*x509.Certificate{parsePEMCert(leafA)}, + }, false) + test("DANE-EE, self-signed", []dns.TLSA{ + singleTlsaRecord(3, 1, 1, keySHA256(rootA)), + }, tls.ConnectionState{ + HandshakeComplete: true, + PeerCertificates: []*x509.Certificate{parsePEMCert(rootA)}, + }, false) + test("DANE-TA, intermediate TA", []dns.TLSA{ + singleTlsaRecord(2, 1, 1, keySHA256(intermediateA)), + }, tls.ConnectionState{ + HandshakeComplete: true, + PeerCertificates: []*x509.Certificate{ + parsePEMCert(leafA), + parsePEMCert(intermediateA), + parsePEMCert(rootA), + }, + }, false) + test("DANE-TA, intermediate TA, mismatch", []dns.TLSA{ + singleTlsaRecord(2, 1, 1, keySHA256(intermediateA)), + }, tls.ConnectionState{ + HandshakeComplete: true, + PeerCertificates: []*x509.Certificate{ + parsePEMCert(leafB), + parsePEMCert(rootB), + }, + }, true) + test("DANE-TA, intermediate TA, multiple records", []dns.TLSA{ + singleTlsaRecord(2, 1, 1, keySHA256(rootB)), + singleTlsaRecord(2, 1, 1, keySHA256(intermediateA)), + // Add multiple times to make sure that multiple records matching the + // same cert do not break anything. + singleTlsaRecord(2, 1, 1, keySHA256(intermediateA)), + }, tls.ConnectionState{ + HandshakeComplete: true, + PeerCertificates: []*x509.Certificate{ + parsePEMCert(leafA), + parsePEMCert(intermediateA), + parsePEMCert(rootA), + }, + }, false) +} diff --git a/internal/target/remote/debugflags.go b/internal/target/remote/debugflags.go new file mode 100644 index 0000000..0a71a10 --- /dev/null +++ b/internal/target/remote/debugflags.go @@ -0,0 +1,35 @@ +//go:build debugflags +// +build debugflags + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + maddycli "github.com/foxcpp/maddy/internal/cli" + "github.com/urfave/cli/v2" +) + +func init() { + maddycli.AddGlobalFlag(&cli.StringFlag{ + Name: "debug.smtpport", + Usage: "SMTP port to use for connections in tests", + Destination: &smtpPort, + }) +} diff --git a/internal/target/remote/metrics.go b/internal/target/remote/metrics.go new file mode 100644 index 0000000..bac88da --- /dev/null +++ b/internal/target/remote/metrics.go @@ -0,0 +1,46 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import "github.com/prometheus/client_golang/prometheus" + +var mxLevelCnt = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "remote", + Name: "conns_mx_level", + Help: "Outbound connections established with specific MX security level", + }, + []string{"module", "level"}, +) + +var tlsLevelCnt = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "maddy", + Subsystem: "remote", + Name: "conns_tls_level", + Help: "Outbound connections established with specific TLS security level", + }, + []string{"module", "level"}, +) + +func init() { + prometheus.MustRegister(mxLevelCnt) + prometheus.MustRegister(tlsLevelCnt) +} diff --git a/internal/target/remote/mxauth_test.go b/internal/target/remote/mxauth_test.go new file mode 100644 index 0000000..2bd8d2d --- /dev/null +++ b/internal/target/remote/mxauth_test.go @@ -0,0 +1,642 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "context" + "crypto/tls" + "errors" + "net" + "strconv" + "testing" + + "github.com/emersion/go-smtp" + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/go-mtasts" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestRemoteDelivery_AuthMX_MTASTS(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + // Testing policy is enough. + Mode: mtasts.ModeTesting, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + }) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_MTASTS_SkipNonMatching(t *testing.T) { + _, be1, srv1 := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.2:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{ + {Host: "mx2.example.invalid.", Pref: 5}, + {Host: "mx1.example.invalid.", Pref: 10}, + }, + }, + "mx1.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "mx2.example.invalid.": { + A: []string{"127.0.0.2"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + Mode: mtasts.ModeEnforce, + MX: []string{"mx2.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + &localPolicy{minMXLevel: module.MX_MTASTS}, + }) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) + + if be1.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_AuthMX_MTASTS_Fail(t *testing.T) { + clientCfg, be1, srv1 := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + Mode: mtasts.ModeTesting, + MX: []string{"mx4.example.invalid"}, // not mx.example.invalid! + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + &localPolicy{minMXLevel: module.MX_MTASTS}, + }) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Fatal("Expected an error, got none") + } + + if be1.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_AuthMX_MTASTS_NoTLS(t *testing.T) { + be1, srv1 := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + Mode: mtasts.ModeEnforce, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + &localPolicy{minMXLevel: module.MX_MTASTS}, + }) + defer tgt.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Fatal("Expected an error, got none") + } + + if be1.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_AuthMX_MTASTS_RequirePKIX(t *testing.T) { + _, be1, srv1 := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + Mode: mtasts.ModeEnforce, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + &localPolicy{minMXLevel: module.MX_MTASTS}, + }) + defer tgt.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Fatal("Expected an error, got none") + } + + if be1.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_AuthMX_MTASTS_NoPolicy(t *testing.T) { + // At the moment, implementation ensures all MX policy checks are completed + // before attempting to connect. + // However, we cannot run complete go-smtp server to check whether it is + // violated and the connection is actually estabilished since this causes + // weird race conditions when test completes before go-smtp has the + // chance to fully initialize itself (Serve is still at the conn.listeners + // assignment when Close is called). + // + // The issue was resolved upstream by introducing locking around internal + // listeners slice use. Uses of FailOnConn remain since they pretty much do + // not hurt. + // + // https://builds.sr.ht/~emersion/job/147975 + tarpit := testutils.FailOnConn(t, "127.0.0.1:"+smtpPort) + defer tarpit.Close() + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return nil, mtasts.ErrNoPolicy + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + &localPolicy{minMXLevel: module.MX_MTASTS}, + }) + defer tgt.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Fatal("Expected an error, got none") + } +} + +func TestRemoteDelivery_AuthMX_DNSSEC(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + AD: true, + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + dnsSrv, err := mockdns.NewServerWithLogger(zones, testutils.Logger(t, "mockdns"), false) + if err != nil { + t.Fatal(err) + } + defer dnsSrv.Close() + + dialer := net.Dialer{} + dialer.Resolver = &net.Resolver{} + dnsSrv.PatchNet(dialer.Resolver) + addr := dnsSrv.LocalAddr().(*net.UDPAddr) + + extResolver, err := dns.NewExtResolver() + if err != nil { + t.Fatal(err) + } + extResolver.Cfg.Servers = []string{addr.IP.String()} + extResolver.Cfg.Port = strconv.Itoa(addr.Port) + + tgt := testTarget(t, zones, extResolver, nil) + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_AuthMX_DNSSEC_Fail(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + dnsSrv, err := mockdns.NewServerWithLogger(zones, testutils.Logger(t, "mockdns"), false) + if err != nil { + t.Fatal(err) + } + defer dnsSrv.Close() + + dialer := net.Dialer{} + dialer.Resolver = &net.Resolver{} + dnsSrv.PatchNet(dialer.Resolver) + addr := dnsSrv.LocalAddr().(*net.UDPAddr) + + extResolver, err := dns.NewExtResolver() + if err != nil { + t.Fatal(err) + } + extResolver.Cfg.Servers = []string{addr.IP.String()} + extResolver.Cfg.Port = strconv.Itoa(addr.Port) + + tgt := testTarget(t, zones, extResolver, []module.MXAuthPolicy{ + &localPolicy{minMXLevel: module.MX_DNSSEC}, + }) + defer tgt.Close() + + _, err = testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Fatal("Expected an error, got none") + } + + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_REQUIRETLS(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + srv.EnableREQUIRETLS = true + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + // Testing policy is enough. + Mode: mtasts.ModeTesting, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + }) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + testutils.DoTestDeliveryMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ + OriginalFrom: "test@example.com", + SMTPOpts: smtp.MailOptions{ + RequireTLS: true, + }, + }) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_REQUIRETLS_Fail(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + srv.EnableREQUIRETLS = false /* no REQUIRETLS */ + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + // Testing policy is enough. + Mode: mtasts.ModeTesting, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + }) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + if _, err := testutils.DoTestDeliveryErrMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ + OriginalFrom: "test@example.com", + SMTPOpts: smtp.MailOptions{ + RequireTLS: true, + }, + }); err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_REQUIRETLS_Relaxed(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + srv.EnableREQUIRETLS = false /* no REQUIRETLS */ + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + // Testing policy is enough. + Mode: mtasts.ModeTesting, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + }) + tgt.relaxedREQUIRETLS = true + tgt.tlsConfig = clientCfg + defer tgt.Close() + + testutils.DoTestDeliveryMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ + OriginalFrom: "test@example.com", + SMTPOpts: smtp.MailOptions{ + RequireTLS: true, + }, + }) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_REQUIRETLS_Relaxed_NoMXAuth(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + srv.EnableREQUIRETLS = false /* no REQUIRETLS */ + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + return nil, mtasts.ErrNoPolicy + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + }) + tgt.relaxedREQUIRETLS = true + tgt.tlsConfig = clientCfg + defer tgt.Close() + + if _, err := testutils.DoTestDeliveryErrMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ + OriginalFrom: "test@example.com", + SMTPOpts: smtp.MailOptions{ + RequireTLS: true, + }, + }); err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_REQUIRETLS_Relaxed_NoTLS(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + srv.EnableREQUIRETLS = false /* no REQUIRETLS */ + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + // Testing policy is enough. + Mode: mtasts.ModeTesting, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + }) + tgt.relaxedREQUIRETLS = true + tgt.tlsConfig = nil + defer tgt.Close() + + if _, err := testutils.DoTestDeliveryErrMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ + OriginalFrom: "test@example.com", + SMTPOpts: smtp.MailOptions{ + RequireTLS: true, + }, + }); err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} + +func TestRemoteDelivery_REQUIRETLS_Relaxed_TLSFail(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + srv.EnableREQUIRETLS = false /* no REQUIRETLS */ + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + mtastsGet := func(_ context.Context, domain string) (*mtasts.Policy, error) { + if domain != "example.invalid" { + return nil, errors.New("Wrong domain in lookup") + } + + return &mtasts.Policy{ + // Testing policy is enough. + Mode: mtasts.ModeTesting, + MX: []string{"mx.example.invalid"}, + }, nil + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + testSTSPolicy(t, zones, mtastsGet), + }) + tgt.relaxedREQUIRETLS = true + // Cause failure through version incompatibility. + clientCfg.MaxVersion = tls.VersionTLS12 + clientCfg.MinVersion = tls.VersionTLS12 + srv.TLSConfig.MinVersion = tls.VersionTLS11 + srv.TLSConfig.MaxVersion = tls.VersionTLS11 + tgt.tlsConfig = clientCfg + defer tgt.Close() + + if _, err := testutils.DoTestDeliveryErrMeta(t, tgt, "test@example.com", []string{"test@example.invalid"}, &module.MsgMetadata{ + OriginalFrom: "test@example.com", + SMTPOpts: smtp.MailOptions{ + RequireTLS: true, + }, + }); err == nil { + t.Error("Expected an error, got none") + } + if be.MailFromCounter != 0 { + t.Fatal("MAIL FROM issued for server failing authentication") + } +} diff --git a/internal/target/remote/policy_group.go b/internal/target/remote/policy_group.go new file mode 100644 index 0000000..68a2202 --- /dev/null +++ b/internal/target/remote/policy_group.go @@ -0,0 +1,105 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/module" +) + +// PolicyGroup is a module container for a group of Policy implementations. +// +// It allows to share a set of policy configurations between remote target +// instances using named configuration blocks (module instances) system. +// +// It is registered globally under the name 'mx_auth'. This is also the name of +// corresponding remote target configuration directive. The object does not +// implement any standard module interfaces besides module.Module and is +// specific to the remote target. +type PolicyGroup struct { + L []module.MXAuthPolicy + instName string + pols map[string]module.MXAuthPolicy +} + +func (pg *PolicyGroup) Init(cfg *config.Map) error { + var debugLog bool + cfg.Bool("debug", true, false, &debugLog) + cfg.AllowUnknown() + other, err := cfg.Process() + if err != nil { + return err + } + + // Policies have defined application order since some of them depend on + // results of other policies. We first initialize them in the order they + // are defined in and then reorder depending on the needed order. + + for _, block := range other { + if _, ok := pg.pols[block.Name]; ok { + return config.NodeErr(block, "duplicate policy block: %v", block.Name) + } + + var policy module.MXAuthPolicy + err := modconfig.ModuleFromNode("mx_auth", append([]string{block.Name}, block.Args...), block, cfg.Globals, &policy) + if err != nil { + return err + } + + pg.pols[block.Name] = policy + } + + for _, name := range [...]string{ + "mtasts", + // sts_preload should go after mtasts so it will take not effect if + // MXLevel is already MX_MTASTS. + "sts_preload", + "dane", + "dnssec", + // localPolicy should be the last one, since it considers levels defined by + // other policies. + "local_policy", + } { + policy, ok := pg.pols[name] + if !ok { + continue + } + pg.L = append(pg.L, policy) + } + + return nil +} + +func (PolicyGroup) Name() string { + return "mx_auth" +} + +func (pg PolicyGroup) InstanceName() string { + return pg.instName +} + +func init() { + module.Register("mx_auth", func(_, instName string, _, _ []string) (module.Module, error) { + return &PolicyGroup{ + instName: instName, + pols: map[string]module.MXAuthPolicy{}, + }, nil + }) +} diff --git a/internal/target/remote/remote.go b/internal/target/remote/remote.go new file mode 100644 index 0000000..d4c42ed --- /dev/null +++ b/internal/target/remote/remote.go @@ -0,0 +1,484 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package remote implements module which does outgoing +// message delivery using servers discovered using DNS MX records. +// +// Implemented interfaces: +// - module.DeliveryTarget +package remote + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "runtime/trace" + "strings" + "sync" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/address" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + tls2 "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/limits" + "github.com/foxcpp/maddy/internal/smtpconn/pool" + "github.com/foxcpp/maddy/internal/target" + "golang.org/x/net/idna" +) + +var smtpPort = "25" + +func moduleError(err error) error { + return exterrors.WithFields(err, map[string]interface{}{ + "target": "remote", + }) +} + +type Target struct { + name string + hostname string + localIP string + ipv4 bool + tlsConfig *tls.Config + + resolver dns.Resolver + dialer func(ctx context.Context, network, addr string) (net.Conn, error) + extResolver *dns.ExtResolver + + policies []module.MXAuthPolicy + limits *limits.Group + allowSecOverride bool + relaxedREQUIRETLS bool + + pool *pool.P + connReuseLimit int + + Log log.Logger + + connectTimeout time.Duration + commandTimeout time.Duration + submissionTimeout time.Duration +} + +var _ module.DeliveryTarget = &Target{} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, errors.New("remote: inline arguments are not used") + } + // Keep this synchronized with testTarget. + return &Target{ + name: instName, + resolver: dns.DefaultResolver(), + dialer: (&net.Dialer{}).DialContext, + Log: log.Logger{Name: "remote"}, + }, nil +} + +func (rt *Target) Init(cfg *config.Map) error { + var err error + rt.extResolver, err = dns.NewExtResolver() + if err != nil { + rt.Log.Error("cannot initialize DNSSEC-aware resolver, DNSSEC and DANE are not available", err) + } + + cfg.String("hostname", true, true, "", &rt.hostname) + cfg.String("local_ip", false, false, "", &rt.localIP) + cfg.Bool("force_ipv4", false, false, &rt.ipv4) + cfg.Bool("debug", true, false, &rt.Log.Debug) + cfg.Custom("tls_client", true, false, func() (interface{}, error) { + return &tls.Config{}, nil + }, tls2.TLSClientBlock, &rt.tlsConfig) + cfg.Custom("mx_auth", false, false, func() (interface{}, error) { + // Default is "no policies" to follow the principles of explicit + // configuration (if it is not requested - it is not done). + return nil, nil + }, func(cfg *config.Map, n config.Node) (interface{}, error) { + // Module instance is &PolicyGroup. + var p *PolicyGroup + if err := modconfig.GroupFromNode("mx_auth", n.Args, n, cfg.Globals, &p); err != nil { + return nil, err + } + return p.L, nil + }, &rt.policies) + cfg.Custom("limits", false, false, func() (interface{}, error) { + return &limits.Group{}, nil + }, func(cfg *config.Map, n config.Node) (interface{}, error) { + var g *limits.Group + if err := modconfig.GroupFromNode("limits", n.Args, n, cfg.Globals, &g); err != nil { + return nil, err + } + return g, nil + }, &rt.limits) + cfg.Bool("requiretls_override", false, true, &rt.allowSecOverride) + cfg.Bool("relaxed_requiretls", false, true, &rt.relaxedREQUIRETLS) + cfg.Int("conn_reuse_limit", false, false, 10, &rt.connReuseLimit) + cfg.Duration("connect_timeout", false, false, 5*time.Minute, &rt.connectTimeout) + cfg.Duration("command_timeout", false, false, 5*time.Minute, &rt.commandTimeout) + cfg.Duration("submission_timeout", false, false, 5*time.Minute, &rt.submissionTimeout) + + poolCfg := pool.Config{ + MaxKeys: 5000, + MaxConnsPerKey: 5, // basically, max. amount of idle connections in cache + MaxConnLifetimeSec: 150, // 2.5 mins, half of recommended idle time from RFC 5321 + StaleKeyLifetimeSec: 60 * 5, // should be bigger than MaxConnLifetimeSec + } + cfg.Int("conn_max_idle_count", false, false, 5, &poolCfg.MaxConnsPerKey) + cfg.Int64("conn_max_idle_time", false, false, 150, &poolCfg.MaxConnLifetimeSec) + + if _, err := cfg.Process(); err != nil { + return err + } + rt.pool = pool.New(poolCfg) + + // INTERNATIONALIZATION: See RFC 6531 Section 3.7.1. + rt.hostname, err = idna.ToASCII(rt.hostname) + if err != nil { + return fmt.Errorf("remote: cannot represent the hostname as an A-label name: %w", err) + } + + if rt.localIP != "" { + addr, err := net.ResolveTCPAddr("tcp", rt.localIP+":0") + if err != nil { + return fmt.Errorf("remote: failed to parse local IP: %w", err) + } + rt.dialer = (&net.Dialer{ + LocalAddr: addr, + }).DialContext + } + if rt.ipv4 { + dial := rt.dialer + rt.dialer = func(ctx context.Context, network, addr string) (net.Conn, error) { + if network == "tcp" { + network = "tcp4" + } + return dial(ctx, network, addr) + } + } + + return nil +} + +func (rt *Target) Close() error { + rt.pool.Close() + + return nil +} + +func (rt *Target) Name() string { + return "remote" +} + +func (rt *Target) InstanceName() string { + return rt.name +} + +type remoteDelivery struct { + rt *Target + mailFrom string + msgMeta *module.MsgMetadata + Log log.Logger + + recipients []string + connections map[string]*mxConn + + policies []module.DeliveryMXAuthPolicy +} + +func (rt *Target) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + policies := make([]module.DeliveryMXAuthPolicy, 0, len(rt.policies)) + if !(msgMeta.TLSRequireOverride && rt.allowSecOverride) { + for _, p := range rt.policies { + policies = append(policies, p.Start(msgMeta)) + } + } + + var ( + ratelimitDomain string + err error + ) + // This will leave ratelimitDomain = "" for null return path which is fine + // for purposes of ratelimiting. + if mailFrom != "" { + _, ratelimitDomain, err = address.Split(mailFrom) + if err != nil { + return nil, &exterrors.SMTPError{ + Code: 501, + EnhancedCode: exterrors.EnhancedCode{5, 1, 8}, + Message: "Malformed sender address", + TargetName: "remote", + Err: err, + } + } + } + + // Domain is already should be normalized by the message source (e.g. + // endpoint/smtp). + region := trace.StartRegion(ctx, "remote/limits.Take") + addr := net.IPv4(127, 0, 0, 1) + if msgMeta.Conn != nil && msgMeta.Conn.RemoteAddr != nil { + tcpAddr, ok := msgMeta.Conn.RemoteAddr.(*net.TCPAddr) + if ok { + addr = tcpAddr.IP + } + } + if err := rt.limits.TakeMsg(ctx, addr, ratelimitDomain); err != nil { + region.End() + return nil, &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 4, 5}, + Message: "High load, try again later", + Reason: "Global limit timeout", + TargetName: "remote", + Err: err, + } + } + region.End() + + return &remoteDelivery{ + rt: rt, + mailFrom: mailFrom, + msgMeta: msgMeta, + Log: target.DeliveryLogger(rt.Log, msgMeta), + connections: map[string]*mxConn{}, + policies: policies, + }, nil +} + +func (rd *remoteDelivery) AddRcpt(ctx context.Context, to string, opts smtp.RcptOptions) error { + defer trace.StartRegion(ctx, "remote/AddRcpt").End() + + if rd.msgMeta.Quarantine { + return &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Refusing to deliver a quarantined message", + TargetName: "remote", + } + } + + _, domain, err := address.Split(to) + if err != nil { + return err + } + + // Special-case for address. If it is not handled by a rewrite rule before + // - we should not attempt to do anything with it and reject it as invalid. + if domain == "" { + return &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 1, 1}, + Message: " address it no supported", + TargetName: "remote", + } + } + + if strings.HasPrefix(domain, "[") { + return &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 1, 1}, + Message: "IP address literals are not supported", + TargetName: "remote", + } + } + + conn, err := rd.connectionForDomain(ctx, domain) + if err != nil { + return err + } + + if err := conn.Rcpt(ctx, to, opts); err != nil { + return moduleError(err) + } + conn.lastUseAt = time.Now() + + rd.recipients = append(rd.recipients, to) + return nil +} + +type multipleErrs struct { + errs map[string]error + statusLck sync.Mutex +} + +func (m *multipleErrs) Error() string { + m.statusLck.Lock() + defer m.statusLck.Unlock() + return fmt.Sprintf("Partial delivery failure, per-rcpt info: %+v", m.errs) +} + +func (m *multipleErrs) Fields() map[string]interface{} { + m.statusLck.Lock() + defer m.statusLck.Unlock() + + // If there are any temporary errors - the sender should retry to make sure + // all recipients will get the message. However, since we can't tell it + // which recipients got the message, this will generate duplicates for + // them. + // + // We favor delivery with duplicates over incomplete delivery here. + + var ( + code = 550 + enchCode = exterrors.EnhancedCode{5, 0, 0} + ) + for _, err := range m.errs { + if exterrors.IsTemporary(err) { + code = 451 + enchCode = exterrors.EnhancedCode{4, 0, 0} + } + } + + return map[string]interface{}{ + "smtp_code": code, + "smtp_enchcode": enchCode, + "smtp_msg": "Partial delivery failure, additional attempts may result in duplicates", + "target": "remote", + "errs": m.errs, + } +} + +func (m *multipleErrs) SetStatus(rcptTo string, err error) { + m.statusLck.Lock() + defer m.statusLck.Unlock() + m.errs[rcptTo] = err +} + +func (rd *remoteDelivery) Body(ctx context.Context, header textproto.Header, buffer buffer.Buffer) error { + defer trace.StartRegion(ctx, "remote/Body").End() + + merr := multipleErrs{ + errs: make(map[string]error), + } + rd.BodyNonAtomic(ctx, &merr, header, buffer) + + for _, v := range merr.errs { + if v != nil { + if len(merr.errs) == 1 { + return v + } + return &merr + } + } + return nil +} + +func (rd *remoteDelivery) BodyNonAtomic(ctx context.Context, c module.StatusCollector, header textproto.Header, b buffer.Buffer) { + defer trace.StartRegion(ctx, "remote/BodyNonAtomic").End() + + if rd.msgMeta.Quarantine { + for _, rcpt := range rd.recipients { + c.SetStatus(rcpt, &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Refusing to deliver quarantined message", + TargetName: "remote", + }) + } + return + } + + var wg sync.WaitGroup + + for i, conn := range rd.connections { + wg.Add(1) + go func() { + defer wg.Done() + + bodyR, err := b.Open() + if err != nil { + for _, rcpt := range conn.Rcpts() { + c.SetStatus(rcpt, err) + } + return + } + defer bodyR.Close() + + err = conn.Data(ctx, header, bodyR) + for _, rcpt := range conn.Rcpts() { + c.SetStatus(rcpt, err) + } + rd.connections[i].errored = err != nil + conn.lastUseAt = time.Now() + }() + } + + wg.Wait() +} + +func (rd *remoteDelivery) Abort(ctx context.Context) error { + return rd.Close() +} + +func (rd *remoteDelivery) Commit(ctx context.Context) error { + // It is not possible to implement it atomically, so users of remoteDelivery have to + // take care of partial failures. + return rd.Close() +} + +func (rd *remoteDelivery) Close() error { + for _, conn := range rd.connections { + rd.rt.limits.ReleaseDest(conn.domain) + conn.transactions++ + + if !conn.Usable() { + rd.Log.Debugf("disconnected %v from %s (errored=%v,transactions=%v,disconnected before=%v)", + conn.LocalAddr(), conn.ServerName(), conn.errored, conn.transactions, conn.C.Client() == nil) + conn.Close() + } else { + rd.Log.Debugf("returning connection %v for %s to pool", conn.LocalAddr(), conn.ServerName()) + rd.rt.pool.Return(conn.domain, conn) + } + } + + var ( + ratelimitDomain string + err error + ) + if rd.mailFrom != "" { + _, ratelimitDomain, err = address.Split(rd.mailFrom) + if err != nil { + return err + } + } + + addr := net.IPv4(127, 0, 0, 1) + if rd.msgMeta.Conn != nil && rd.msgMeta.Conn.RemoteAddr != nil { + tcpAddr, ok := rd.msgMeta.Conn.RemoteAddr.(*net.TCPAddr) + if ok { + addr = tcpAddr.IP + } + } + rd.rt.limits.ReleaseMsg(addr, ratelimitDomain) + + return nil +} + +func init() { + module.Register("target.remote", New) +} diff --git a/internal/target/remote/remote_test.go b/internal/target/remote/remote_test.go new file mode 100644 index 0000000..4998e0c --- /dev/null +++ b/internal/target/remote/remote_test.go @@ -0,0 +1,1054 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "context" + "crypto/tls" + "flag" + "math/rand" + "net" + "os" + "strconv" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/go-mtasts" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/limits" + "github.com/foxcpp/maddy/internal/smtpconn/pool" + "github.com/foxcpp/maddy/internal/testutils" +) + +// .invalid TLD is used here to make sure if there is something wrong about +// DNS hooks and lookups go to the real Internet, they will not result in +// any useful data that can lead to outgoing connections being made. + +func testTarget(t *testing.T, zones map[string]mockdns.Zone, extResolver *dns.ExtResolver, + extraPolicies []module.MXAuthPolicy) *Target { + resolver := &mockdns.Resolver{Zones: zones} + + tgt := Target{ + name: "remote", + hostname: "mx.example.com", + resolver: resolver, + dialer: resolver.DialContext, + extResolver: extResolver, + tlsConfig: &tls.Config{}, + Log: testutils.Logger(t, "remote"), + policies: extraPolicies, + limits: &limits.Group{}, + pool: pool.New(pool.Config{ + MaxKeys: 5000, + MaxConnsPerKey: 5, // basically, max. amount of idle connections in cache + MaxConnLifetimeSec: 150, // 2.5 mins, half of recommended idle time from RFC 5321 + StaleKeyLifetimeSec: 60 * 5, // should be bigger than MaxConnLifetimeSec + }), + } + + return &tgt +} + +func testSTSPolicy(t *testing.T, zones map[string]mockdns.Zone, mtastsGet func(context.Context, string) (*mtasts.Policy, error)) *mtastsPolicy { + m, err := NewMTASTSPolicy("mx_auth.mtasts", "test", nil, nil) + if err != nil { + t.Fatal(err) + } + p := m.(*mtastsPolicy) + err = p.Init(config.NewMap(nil, config.Node{ + Children: []config.Node{ + { + Name: "cache", + Args: []string{"ram"}, + }, + }, + })) + if err != nil { + t.Fatal(err) + } + + p.mtastsGet = mtastsGet + p.log = testutils.Logger(t, "remote/mtasts") + p.cache.Resolver = &mockdns.Resolver{Zones: zones} + p.StartUpdater() + + return p +} + +func testDANEPolicy(t *testing.T, extR *dns.ExtResolver) *danePolicy { + m, err := NewDANEPolicy("mx_auth.dane", "test", nil, nil) + if err != nil { + t.Fatal(err) + } + p := m.(*danePolicy) + err = p.Init(config.NewMap(nil, config.Node{ + Children: nil, + })) + if err != nil { + t.Fatal(err) + } + + p.extResolver = extR + p.log = testutils.Logger(t, "remote/dane") + return p +} + +func TestRemoteDelivery(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_NoMXFallback(t *testing.T) { + tarpit := testutils.FailOnConn(t, "127.0.0.1:"+smtpPort) + defer tarpit.Close() + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + if err := delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}); err == nil { + t.Fatal("Expected an error, got none") + } + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_EmptySender(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + testutils.DoTestDelivery(t, tgt, "", []string{"test@example.invalid"}) + + be.CheckMsg(t, 0, "", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_IPLiteral(t *testing.T) { + t.Skip("Support disabled") + + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "1.0.0.127.in-addr.arpa.": { + PTR: []string{"mx.example.invalid."}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@[127.0.0.1]"}) + + be.CheckMsg(t, 0, "test@example.com", []string{"test@[127.0.0.1]"}) +} + +func TestRemoteDelivery_FallbackMX(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_BodyNonAtomic(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + c := multipleErrs{ + errs: map[string]error{}, + } + testutils.DoTestDeliveryNonAtomic(t, &c, tgt, "test@example.com", []string{"test@example.invalid"}) + + if err := c.errs["test@example.invalid"]; err != nil { + t.Fatal(err) + } + + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_Abort(t *testing.T) { + _, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + if err := delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_CommitWithoutBody(t *testing.T) { + _, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + if err := delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + // Currently it does nothing, probably it should fail. + if err := delivery.Commit(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_MAILFROMErr(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + be.MailErr = &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + err = delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}) + testutils.CheckSMTPErr(t, err, 550, exterrors.EnhancedCode{5, 1, 2}, "mx.example.invalid. said: Hey") + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_NoMX(t *testing.T) { + tarpit := testutils.FailOnConn(t, "127.0.0.1:"+smtpPort) + defer tarpit.Close() + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + if err := delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}); err == nil { + t.Fatal("Expected an error, got none") + } + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_NullMX(t *testing.T) { + // Hang the test if it actually connects to the server to + // deliver the message. Use of testutils.SMTPServer here + // causes weird race conditions. + tarpit := testutils.FailOnConn(t, "127.0.0.1:"+smtpPort) + defer tarpit.Close() + + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: ".", Pref: 10}}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + err = delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}) + testutils.CheckSMTPErr(t, err, 556, exterrors.EnhancedCode{5, 1, 10}, "Domain does not accept email (null MX)") + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_Quarantined(t *testing.T) { + _, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + meta := module.MsgMetadata{ID: "test..."} + + delivery, err := tgt.Start(context.Background(), &meta, "test@example.com") + if err != nil { + t.Fatal(err) + } + + if err := delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + meta.Quarantine = true + + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + body := buffer.MemoryBuffer{Slice: []byte("foobar\n")} + if err := delivery.Body(context.Background(), textproto.Header{}, body); err == nil { + t.Fatal("Expected an error, got none") + } + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_MAILFROMErr_Repeated(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + be.MailErr = &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + err = delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}) + testutils.CheckSMTPErr(t, err, 550, exterrors.EnhancedCode{5, 1, 2}, "mx.example.invalid. said: Hey") + + err = delivery.AddRcpt(context.Background(), "test2@example.invalid", smtp.RcptOptions{}) + testutils.CheckSMTPErr(t, err, 550, exterrors.EnhancedCode{5, 1, 2}, "mx.example.invalid. said: Hey") + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_RcptErr(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + be.RcptErr = map[string]error{ + "test@example.invalid": &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + err = delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}) + testutils.CheckSMTPErr(t, err, 550, exterrors.EnhancedCode{5, 1, 2}, "mx.example.invalid. said: Hey") + + // It should be possible to, however, add another recipient and continue + // delivery as if nothing happened. + if err := delivery.AddRcpt(context.Background(), "test2@example.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + body := buffer.MemoryBuffer{Slice: []byte("foobar\n")} + if err := delivery.Body(context.Background(), hdr, body); err != nil { + t.Fatal(err) + } + + if err := delivery.Commit(context.Background()); err != nil { + t.Fatal(err) + } + + be.CheckMsg(t, 0, "test@example.com", []string{"test2@example.invalid"}) +} + +func TestRemoteDelivery_DownMX(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{ + {Host: "mx1.example.invalid.", Pref: 20}, + {Host: "mx2.example.invalid.", Pref: 10}, + }, + }, + "mx1.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "mx2.example.invalid.": { + A: []string{"127.0.0.2"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_AllMXDown(t *testing.T) { + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{ + {Host: "mx1.example.invalid.", Pref: 20}, + {Host: "mx2.example.invalid.", Pref: 10}, + }, + }, + "mx1.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "mx2.example.invalid.": { + A: []string{"127.0.0.2"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Fatal("Expected an error, got none") + } +} + +func TestRemoteDelivery_Split(t *testing.T) { + be1, srv1 := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + be2, srv2 := testutils.SMTPServer(t, "127.0.0.2:"+smtpPort) + defer srv2.Close() + defer testutils.CheckSMTPConnLeak(t, srv2) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "example2.invalid.": { + MX: []net.MX{{Host: "mx.example2.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "mx.example2.invalid.": { + A: []string{"127.0.0.2"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid", "test@example2.invalid"}) + + be1.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) + be2.CheckMsg(t, 0, "test@example.com", []string{"test@example2.invalid"}) +} + +func TestRemoteDelivery_Split_Fail(t *testing.T) { + be1, srv1 := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + be2, srv2 := testutils.SMTPServer(t, "127.0.0.2:"+smtpPort) + defer srv2.Close() + defer testutils.CheckSMTPConnLeak(t, srv2) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "example2.invalid.": { + MX: []net.MX{{Host: "mx.example2.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "mx.example2.invalid.": { + A: []string{"127.0.0.2"}, + }, + } + + be1.RcptErr = map[string]error{ + "test@example.invalid": &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + }, + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + err = delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}) + if err == nil { + t.Fatal("Expected an error, got none") + } + + // It should be possible to, however, add another recipient and continue + // delivery as if nothing happened. + if err := delivery.AddRcpt(context.Background(), "test@example2.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + body := buffer.MemoryBuffer{Slice: []byte("foobar\n")} + if err := delivery.Body(context.Background(), hdr, body); err != nil { + t.Fatal(err) + } + + if err := delivery.Commit(context.Background()); err != nil { + t.Fatal(err) + } + + be2.CheckMsg(t, 0, "test@example.com", []string{"test@example2.invalid"}) +} + +func TestRemoteDelivery_BodyErr(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + be.DataErr = &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + err = delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}) + if err != nil { + t.Fatal(err) + } + + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + body := buffer.MemoryBuffer{Slice: []byte("foobar\n")} + if err := delivery.Body(context.Background(), hdr, body); err == nil { + t.Fatal("expected an error, got none") + } + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_Split_BodyErr(t *testing.T) { + be1, srv1 := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + _, srv2 := testutils.SMTPServer(t, "127.0.0.2:"+smtpPort) + defer srv2.Close() + defer testutils.CheckSMTPConnLeak(t, srv2) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "example2.invalid.": { + MX: []net.MX{{Host: "mx.example2.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "mx.example2.invalid.": { + A: []string{"127.0.0.2"}, + }, + } + + be1.DataErr = &smtp.SMTPError{ + Code: 421, + EnhancedCode: smtp.EnhancedCode{4, 1, 2}, + Message: "Hey", + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + if err := delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + if err := delivery.AddRcpt(context.Background(), "test@example2.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + body := buffer.MemoryBuffer{Slice: []byte("foobar\n")} + err = delivery.Body(context.Background(), hdr, body) + testutils.CheckSMTPErr(t, err, 451, exterrors.EnhancedCode{4, 0, 0}, + "Partial delivery failure, additional attempts may result in duplicates") + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_Split_BodyErr_NonAtomic(t *testing.T) { + be1, srv1 := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv1.Close() + defer testutils.CheckSMTPConnLeak(t, srv1) + _, srv2 := testutils.SMTPServer(t, "127.0.0.2:"+smtpPort) + defer srv2.Close() + defer testutils.CheckSMTPConnLeak(t, srv2) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "example2.invalid.": { + MX: []net.MX{{Host: "mx.example2.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + "mx.example2.invalid.": { + A: []string{"127.0.0.2"}, + }, + } + + be1.DataErr = &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + } + + tgt := testTarget(t, zones, nil, nil) + defer tgt.Close() + + delivery, err := tgt.Start(context.Background(), &module.MsgMetadata{ID: "test..."}, "test@example.com") + if err != nil { + t.Fatal(err) + } + + if err := delivery.AddRcpt(context.Background(), "test@example.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + if err := delivery.AddRcpt(context.Background(), "test2@example.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + if err := delivery.AddRcpt(context.Background(), "test@example2.invalid", smtp.RcptOptions{}); err != nil { + t.Fatal(err) + } + + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + body := buffer.MemoryBuffer{Slice: []byte("foobar\n")} + c := multipleErrs{ + errs: map[string]error{}, + } + delivery.(module.PartialDelivery).BodyNonAtomic(context.Background(), &c, hdr, body) + + testutils.CheckSMTPErr(t, c.errs["test@example.invalid"], + 550, exterrors.EnhancedCode{5, 1, 2}, "mx.example.invalid. said: Hey") + testutils.CheckSMTPErr(t, c.errs["test2@example.invalid"], + 550, exterrors.EnhancedCode{5, 1, 2}, "mx.example.invalid. said: Hey") + if err := c.errs["test@example2.invalid"]; err != nil { + t.Errorf("Unexpected error for non-failing connection: %v", err) + } + + if err := delivery.Abort(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDelivery_TLSErrFallback(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + // Cause failure through version incompatibility. + clientCfg.MaxVersion = tls.VersionTLS12 + clientCfg.MinVersion = tls.VersionTLS12 + srv.TLSConfig.MinVersion = tls.VersionTLS11 + srv.TLSConfig.MaxVersion = tls.VersionTLS11 + + tgt := testTarget(t, zones, nil, nil) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_RequireTLS_Missing(t *testing.T) { + _, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + &localPolicy{minTLSLevel: module.TLSEncrypted}, + }) + defer tgt.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Errorf("expected an error, got none") + } +} + +func TestRemoteDelivery_RequireTLS_Present(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + &localPolicy{minTLSLevel: module.TLSEncrypted}, + }) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestRemoteDelivery_RequireTLS_NoErrFallback(t *testing.T) { + clientCfg, _, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + // Cause failure through version incompatibility. + clientCfg.MaxVersion = tls.VersionTLS12 + clientCfg.MinVersion = tls.VersionTLS12 + srv.TLSConfig.MinVersion = tls.VersionTLS11 + srv.TLSConfig.MaxVersion = tls.VersionTLS11 + + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + &localPolicy{minTLSLevel: module.TLSEncrypted}, + }) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + _, err := testutils.DoTestDeliveryErr(t, tgt, "test@example.com", []string{"test@example.invalid"}) + if err == nil { + t.Fatal("Expected an error, got none") + } +} + +func TestRemoteDelivery_TLS_FallbackNoVerify(t *testing.T) { + _, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + // tlsConfig is not configured to trust server cert. + tgt := testTarget(t, zones, nil, []module.MXAuthPolicy{ + &localPolicy{minTLSLevel: module.TLSEncrypted}, + }) + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) + + // But it should still be delivered over TLS. + tlsState, ok := be.Messages[0].Conn.TLSConnectionState() + if !ok || !tlsState.HandshakeComplete { + t.Fatal("Message was not delivered over TLS") + } +} + +func TestRemoteDelivery_TLS_FallbackPlaintext(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + // Cause failure through version incompatibility. + clientCfg.MaxVersion = tls.VersionTLS12 + clientCfg.MinVersion = tls.VersionTLS12 + srv.TLSConfig.MinVersion = tls.VersionTLS11 + srv.TLSConfig.MaxVersion = tls.VersionTLS11 + + tgt := testTarget(t, zones, nil, nil) + tgt.tlsConfig = clientCfg + defer tgt.Close() + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) +} + +func TestMain(m *testing.M) { + remoteSmtpPort := flag.String("test.smtpport", "random", "(maddy) SMTP port to use for connections in tests") + flag.Parse() + + if *remoteSmtpPort == "random" { + *remoteSmtpPort = strconv.Itoa(rand.Intn(65536-10000) + 10000) + } + + smtpPort = *remoteSmtpPort + os.Exit(m.Run()) +} + +func TestRemoteDelivery_ConnReuse(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+smtpPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + zones := map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + } + + tgt := testTarget(t, zones, nil, nil) + tgt.connReuseLimit = 5 + defer tgt.Close() + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) + be.CheckMsg(t, 1, "test@example.com", []string{"test@example.invalid"}) + + if len(be.SourceEndpoints) != 1 { + t.Fatal("Only one session should be used, found", be.SourceEndpoints) + } +} diff --git a/internal/target/remote/security.go b/internal/target/remote/security.go new file mode 100644 index 0000000..a8177fb --- /dev/null +++ b/internal/target/remote/security.go @@ -0,0 +1,642 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package remote + +import ( + "context" + "crypto/tls" + "errors" + "os" + "runtime/debug" + "time" + + "github.com/foxcpp/go-mtasts" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/dns" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/future" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/target" +) + +type ( + mtastsPolicy struct { + cache *mtasts.Cache + mtastsGet func(context.Context, string) (*mtasts.Policy, error) + updaterStop chan struct{} + log log.Logger + instName string + } + mtastsDelivery struct { + c *mtastsPolicy + domain string + policyFut *future.Future + log log.Logger + } +) + +func NewMTASTSPolicy(_, instName string, _, _ []string) (module.Module, error) { + return &mtastsPolicy{ + instName: instName, + log: log.Logger{Name: "mx_auth.mtasts", Debug: log.DefaultLogger.Debug}, + }, nil +} + +func (c *mtastsPolicy) Name() string { + return c.log.Name +} + +func (c *mtastsPolicy) InstanceName() string { + return c.instName +} + +func (c *mtastsPolicy) Weight() int { + return 10 +} + +func (c *mtastsPolicy) Init(cfg *config.Map) error { + var ( + storeType string + storeDir string + ) + cfg.Enum("cache", false, false, []string{"ram", "fs"}, "fs", &storeType) + cfg.String("fs_dir", false, false, "mtasts_cache", &storeDir) + if _, err := cfg.Process(); err != nil { + return err + } + + switch storeType { + case "fs": + if err := os.MkdirAll(storeDir, os.ModePerm); err != nil { + return err + } + c.cache = mtasts.NewFSCache(storeDir) + case "ram": + c.cache = mtasts.NewRAMCache() + default: + panic("mtasts policy init: unknown cache type") + } + c.cache.Resolver = dns.DefaultResolver() + c.mtastsGet = c.cache.Get + + return nil +} + +// StartUpdater starts a goroutine to update MTA-STS cache periodically until +// Close is called. +// +// It can be called only once per mtastsPolicy instance. +func (c *mtastsPolicy) StartUpdater() { + c.updaterStop = make(chan struct{}) + go c.updater() +} + +func (c *mtastsPolicy) updater() { + defer func() { + if err := recover(); err != nil { + stack := debug.Stack() + log.Printf("panic during MTA-STS update: %v\n%s", err, stack) + log.Printf("MTA-STS cache refresh disabled due to critical error") + c.updaterStop = nil + } + }() + + // Always update cache on start-up since we may have been down for some + // time. + c.log.Debugln("updating MTA-STS cache...") + if err := c.cache.Refresh(); err != nil { + c.log.Error("MTA-STS cache update error", err) + } + c.log.Debugln("updating MTA-STS cache... done!") + + t := time.NewTicker(12 * time.Hour) + for { + select { + case <-t.C: + c.log.Debugln("updating MTA-STS cache...") + if err := c.cache.Refresh(); err != nil { + c.log.Error("MTA-STS cache opdate error", err) + } + c.log.Debugln("updating MTA-STS cache... done!") + case <-c.updaterStop: + c.updaterStop <- struct{}{} + return + } + } +} + +func (c *mtastsPolicy) Start(msgMeta *module.MsgMetadata) module.DeliveryMXAuthPolicy { + return &mtastsDelivery{ + c: c, + log: target.DeliveryLogger(c.log, msgMeta), + } +} + +func (c *mtastsPolicy) Close() error { + if c.updaterStop != nil { + c.updaterStop <- struct{}{} + <-c.updaterStop + c.updaterStop = nil + } + return nil +} + +func (c *mtastsDelivery) PrepareDomain(ctx context.Context, domain string) { + c.policyFut = future.New() + go func() { + c.policyFut.Set(c.c.mtastsGet(ctx, domain)) + }() +} + +func (c *mtastsDelivery) PrepareConn(ctx context.Context, mx string) {} + +func (c *mtastsDelivery) CheckMX(ctx context.Context, mxLevel module.MXLevel, domain, mx string, dnssec bool) (module.MXLevel, error) { + policyI, err := c.policyFut.GetContext(ctx) + if err != nil { + c.log.DebugMsg("MTA-STS error", "err", err) + return module.MXNone, nil + } + policy := policyI.(*mtasts.Policy) + + if !policy.Match(mx) { + if policy.Mode == mtasts.ModeEnforce { + return module.MXNone, &exterrors.SMTPError{ + Code: 550, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Failed to establish the MX record authenticity (MTA-STS)", + } + } + c.log.Msg("MX does not match published non-enforced MTA-STS policy", "mx", mx, "domain", c.domain) + return module.MXNone, nil + } + return module.MX_MTASTS, nil +} + +func (c *mtastsDelivery) CheckConn(ctx context.Context, mxLevel module.MXLevel, tlsLevel module.TLSLevel, domain, mx string, tlsState tls.ConnectionState) (module.TLSLevel, error) { + policyI, err := c.policyFut.GetContext(ctx) + if err != nil { + c.c.log.DebugMsg("MTA-STS error", "err", err) + return module.TLSNone, nil + } + policy := policyI.(*mtasts.Policy) + + if policy.Mode != mtasts.ModeEnforce { + return module.TLSNone, nil + } + + if !tlsState.HandshakeComplete { + return module.TLSNone, &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 1}, + Message: "TLS is required but unavailable or failed (MTA-STS)", + } + } + + if tlsState.VerifiedChains == nil { + return module.TLSNone, &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 1}, + Message: "Recipient server TLS certificate is not trusted but " + + "authentication is required by MTA-STS", + Misc: map[string]interface{}{ + "tls_level": tlsLevel, + }, + } + } + + return module.TLSNone, nil +} + +func (c *mtastsDelivery) Reset(msgMeta *module.MsgMetadata) { + c.policyFut = nil + if msgMeta != nil { + c.log = target.DeliveryLogger(c.c.log, msgMeta) + } +} + +// Stub that will be removed in 0.5. +type stsPreloadPolicy struct { + log log.Logger + instName string +} + +func NewSTSPreload(_, instName string, _, _ []string) (module.Module, error) { + return &stsPreloadPolicy{ + instName: instName, + log: log.Logger{Name: "mx_auth.sts_preload", Debug: log.DefaultLogger.Debug}, + }, nil +} + +func (c *stsPreloadPolicy) Name() string { + return c.log.Name +} + +func (c *stsPreloadPolicy) InstanceName() string { + return c.instName +} + +func (c *stsPreloadPolicy) Weight() int { + return 30 // after MTA-STS +} + +func (c *stsPreloadPolicy) Init(cfg *config.Map) error { + c.log.Println("sts_preload module is deprecated and is no-op as the list is expired and unmaintained") + + var ( + sourcePath string + enforceTesting bool + ) + cfg.String("source", false, false, "eff", &sourcePath) + cfg.Bool("enforce_testing", false, true, &enforceTesting) + if _, err := cfg.Process(); err != nil { + return err + } + + return nil +} + +type preloadDelivery struct { + *stsPreloadPolicy +} + +func (p *stsPreloadPolicy) Start(*module.MsgMetadata) module.DeliveryMXAuthPolicy { + return &preloadDelivery{stsPreloadPolicy: p} +} + +func (p *preloadDelivery) Reset(*module.MsgMetadata) {} +func (p *preloadDelivery) PrepareDomain(ctx context.Context, domain string) {} +func (p *preloadDelivery) PrepareConn(ctx context.Context, mx string) {} +func (p *preloadDelivery) CheckMX(ctx context.Context, mxLevel module.MXLevel, domain, mx string, dnssec bool) (module.MXLevel, error) { + return mxLevel, nil +} + +func (p *preloadDelivery) CheckConn(ctx context.Context, mxLevel module.MXLevel, tlsLevel module.TLSLevel, domain, mx string, tlsState tls.ConnectionState) (module.TLSLevel, error) { + return tlsLevel, nil +} + +func (p *stsPreloadPolicy) Close() error { + return nil +} + +type dnssecPolicy struct { + instName string +} + +func NewDNSSECPolicy(_, instName string, _, _ []string) (module.Module, error) { + return &dnssecPolicy{ + instName: instName, + }, nil +} + +func (c *dnssecPolicy) Name() string { + return "mx_auth.dnssec" +} + +func (c *dnssecPolicy) InstanceName() string { + return c.instName +} + +func (c *dnssecPolicy) Weight() int { + return 1 +} + +func (c *dnssecPolicy) Init(cfg *config.Map) error { + _, err := cfg.Process() // will fail if there is any directive + return err +} + +func (dnssecPolicy) Start(*module.MsgMetadata) module.DeliveryMXAuthPolicy { + return dnssecPolicy{} +} + +func (dnssecPolicy) Close() error { + return nil +} + +func (dnssecPolicy) Reset(*module.MsgMetadata) {} +func (dnssecPolicy) PrepareDomain(ctx context.Context, domain string) {} +func (dnssecPolicy) PrepareConn(ctx context.Context, mx string) {} + +func (dnssecPolicy) CheckMX(ctx context.Context, mxLevel module.MXLevel, domain, mx string, dnssec bool) (module.MXLevel, error) { + if dnssec { + return module.MX_DNSSEC, nil + } + return module.MXNone, nil +} + +func (dnssecPolicy) CheckConn(ctx context.Context, mxLevel module.MXLevel, tlsLevel module.TLSLevel, domain, mx string, tlsState tls.ConnectionState) (module.TLSLevel, error) { + return module.TLSNone, nil +} + +type ( + danePolicy struct { + extResolver *dns.ExtResolver + log log.Logger + instName string + } + daneDelivery struct { + c *danePolicy + tlsaFut *future.Future + } +) + +func NewDANEPolicy(_, instName string, _, _ []string) (module.Module, error) { + return &danePolicy{ + instName: instName, + log: log.Logger{Name: "remote/dane", Debug: log.DefaultLogger.Debug}, + }, nil +} + +func (c *danePolicy) Name() string { + return "mx_auth.dane" +} + +func (c *danePolicy) InstanceName() string { + return c.instName +} + +func (c *danePolicy) Weight() int { + return 10 +} + +func (c *danePolicy) Init(cfg *config.Map) error { + var err error + c.extResolver, err = dns.NewExtResolver() + if err != nil { + c.log.Error("DANE support is no-op: unable to init EDNS resolver", err) + } + + cfg.Bool("debug", true, log.DefaultLogger.Debug, &c.log.Debug) + + _, err = cfg.Process() + return err +} + +func (c *danePolicy) Start(*module.MsgMetadata) module.DeliveryMXAuthPolicy { + return &daneDelivery{c: c} +} + +func (c *danePolicy) Close() error { + return nil +} + +func (c *daneDelivery) PrepareDomain(ctx context.Context, domain string) {} + +func (c *daneDelivery) discoverTLSA(ctx context.Context, mx string) ([]dns.TLSA, error) { + adA, rname, err := c.c.extResolver.CheckCNAMEAD(ctx, mx) + if err != nil { + // This may indicate a bogus DNSSEC signature or other lookup issue + // (including non-existing domain). + // Per RFC 7672, any I/O errors (including SERVFAIL) should + // cause delivery to be delayed. + return nil, err + } + if rname == "" { + // No A/AAAA records, short-circuit discovery instead of doing useless + // queries. + return nil, errors.New("no address associated with the host") + } + if !adA { + // If A lookup is not DNSSEC-authenticated we assume the server cannot + // have TLSA record and skip trying to actually lookup TLSA + // to avoid hitting weird errors like SERVFAIL, NOTIMP + // e.g. see https://github.com/foxcpp/maddy/issues/287 + if rname == mx { + c.c.log.Debugln("skipping DANE for", mx, "due to non-authenticated A records") + return nil, nil + } + + // But if it is CNAME'd then we may not want to skip it and actually + // consider initial name since it may be signed. To confirm the + // initial name is signed, do CNAME lookup. + cnameAD, _, err := c.c.extResolver.AuthLookupCNAME(ctx, mx) + if err != nil { + return nil, err + } + if !cnameAD { + c.c.log.Debugln("skipping DANE for", mx, "due to non-authenticated CNAME record") + return nil, nil + } + } + + // If there was a CNAME - try it first. + if rname != mx { + ad, recs, err := c.c.extResolver.AuthLookupTLSA(ctx, "25", "tcp", rname) + if err != nil && !dns.IsNotFound(err) { + return nil, err + } + if ad && len(recs) != 0 { + // recs may be empty or contain only unusable records - this is + // okay per RFC 7672, no fallback to initial name is done. + c.c.log.Debugln("using", len(recs), "DANE records at", rname, "to authenticate", mx) + return recs, nil + } + // Per RFC 7672 Section 2.2 we interpret a non-authenticated RRset just + // like an empty RRset and fallback to trying original name. + c.c.log.Debugln("ignoring non-authenticated TLSA records for", rname) + } + + // If initial name is not a CNAME or final canonical name is not "secure" + // - we consider TLSA under the initial name. + ad, recs, err := c.c.extResolver.AuthLookupTLSA(ctx, "25", "tcp", mx) + if err != nil && !dns.IsNotFound(err) { + return nil, err + } + if !ad { + c.c.log.Debugln("ignoring non-authenticated TLSA records for", mx) + return nil, nil + } + + c.c.log.Debugln("using", len(recs), "DANE records at original name to authenticate", mx) + return recs, nil +} + +func (c *daneDelivery) PrepareConn(ctx context.Context, mx string) { + // No DNSSEC support. + if c.c.extResolver == nil { + return + } + + c.tlsaFut = future.New() + + go func() { + defer func() { + if err := recover(); err != nil { + stack := debug.Stack() + log.Printf("panic during extended resolver lookup: %v\n%s", err, stack) + } + }() + + c.tlsaFut.Set(c.discoverTLSA(ctx, dns.FQDN(mx))) + }() +} + +func (c *daneDelivery) CheckMX(ctx context.Context, mxLevel module.MXLevel, domain, mx string, dnssec bool) (module.MXLevel, error) { + return module.MXNone, nil +} + +func (c *daneDelivery) CheckConn(ctx context.Context, mxLevel module.MXLevel, tlsLevel module.TLSLevel, domain, mx string, tlsState tls.ConnectionState) (module.TLSLevel, error) { + // No DNSSEC support. + if c.c.extResolver == nil { + return module.TLSNone, nil + } + + recsI, err := c.tlsaFut.GetContext(ctx) + if err != nil { + // No records. + if dns.IsNotFound(err) { + return module.TLSNone, nil + } + + // Lookup error here indicates a resolution failure or may also + // indicate a bogus DNSSEC signature. + // There is a big problem with differentiating these two. + // + // We assume DANE failure in both cases as a safety measure. + // However, there is a possibility of a temporary error condition, + // so we mark it as such. + return module.TLSNone, exterrors.WithTemporary(err, true) + } + recs := recsI.([]dns.TLSA) + + overridePKIX, err := verifyDANE(recs, tlsState) + if err != nil { + return module.TLSNone, err + } + if overridePKIX { + return module.TLSAuthenticated, nil + } + return module.TLSNone, nil +} + +func (c *daneDelivery) Reset(*module.MsgMetadata) {} + +type ( + localPolicy struct { + instName string + minTLSLevel module.TLSLevel + minMXLevel module.MXLevel + } +) + +func NewLocalPolicy(_, instName string, _, _ []string) (module.Module, error) { + return &localPolicy{ + instName: instName, + }, nil +} + +func (c *localPolicy) Name() string { + return "mx_auth.local_policy" +} + +func (c *localPolicy) InstanceName() string { + return c.instName +} + +func (c *localPolicy) Weight() int { + return 1000 +} + +func (c *localPolicy) Init(cfg *config.Map) error { + var ( + minTLSLevel string + minMXLevel string + ) + + cfg.Enum("min_tls_level", false, false, + []string{"none", "encrypted", "authenticated"}, "encrypted", &minTLSLevel) + cfg.Enum("min_mx_level", false, false, + []string{"none", "mtasts", "dnssec"}, "none", &minMXLevel) + if _, err := cfg.Process(); err != nil { + return err + } + + // Enum checks the value against allowed list, no 'default' necessary. + switch minTLSLevel { + case "none": + c.minTLSLevel = module.TLSNone + case "encrypted": + c.minTLSLevel = module.TLSEncrypted + case "authenticated": + c.minTLSLevel = module.TLSAuthenticated + } + switch minMXLevel { + case "none": + c.minMXLevel = module.MXNone + case "mtasts": + c.minMXLevel = module.MX_MTASTS + case "dnssec": + c.minMXLevel = module.MX_DNSSEC + } + + return nil +} + +func (l localPolicy) Start(msgMeta *module.MsgMetadata) module.DeliveryMXAuthPolicy { + return l +} + +func (l localPolicy) Close() error { + return nil +} + +func (l localPolicy) Reset(*module.MsgMetadata) {} +func (l localPolicy) PrepareDomain(ctx context.Context, domain string) {} +func (l localPolicy) PrepareConn(ctx context.Context, mx string) {} + +func (l localPolicy) CheckMX(ctx context.Context, mxLevel module.MXLevel, domain, mx string, dnssec bool) (module.MXLevel, error) { + if mxLevel < l.minMXLevel { + return module.MXNone, &exterrors.SMTPError{ + // Err on the side of caution if policy evaluation was messed up by + // a temporary error (we can't know with the current design). + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 0}, + Message: "Failed to establish the MX record authenticity", + Misc: map[string]interface{}{ + "mx_level": mxLevel, + "required_mx_level": l.minMXLevel, + }, + } + } + return module.MXNone, nil +} + +func (l localPolicy) CheckConn(ctx context.Context, mxLevel module.MXLevel, tlsLevel module.TLSLevel, domain, mx string, tlsState tls.ConnectionState) (module.TLSLevel, error) { + if tlsLevel < l.minTLSLevel { + return module.TLSNone, &exterrors.SMTPError{ + Code: 451, + EnhancedCode: exterrors.EnhancedCode{4, 7, 1}, + Message: "TLS it not available or unauthenticated but required", + Misc: map[string]interface{}{ + "tls_level": tlsLevel, + "required_tls_level": l.minTLSLevel, + }, + } + } + return module.TLSNone, nil +} + +func init() { + module.Register("mx_auth.mtasts", NewMTASTSPolicy) + module.Register("mx_auth.sts_preload", NewSTSPreload) + module.Register("mx_auth.dnssec", NewDNSSECPolicy) + module.Register("mx_auth.dane", NewDANEPolicy) + module.Register("mx_auth.local_policy", NewLocalPolicy) +} diff --git a/internal/target/skeleton.go b/internal/target/skeleton.go new file mode 100644 index 0000000..00d481c --- /dev/null +++ b/internal/target/skeleton.go @@ -0,0 +1,130 @@ +//go:build ignore +// +build ignore + +// Copy that file into target/ subdirectory. + +package target_name + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2021 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +const modName = "target.target_name" + +type Target struct { + instName string + log log.Logger +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + // If wanted, extract any values from inlineArgs (these values: + // deliver_to target_name ARG1 ARG2 { ... } + + return &Target{ + instName: instName, + log: log.Logger{Name: instName}, + }, nil +} + +func (t *Target) Init(cfg *config.Map) error { + cfg.Bool("debug", true, false, &t.log.Debug) + + // Read any config directives into Target variables here. + + if _, err := cfg.Process(); err != nil { + return err + } + + // Finish setup using obtained values. + + return nil +} + +func (t *Target) Name() string { + return modName +} + +func (t *Target) InstanceName() string { + return t.instName +} + +// If it necessary to have any server shutdown cleanup - implement Close. + +func (t *Target) Close() error { + return nil +} + +type delivery struct { + t *Target + mailFrom string + log log.Logger + msgMeta *module.MsgMetadata +} + +/* +See module.DeliveryTarget and module.Delivery docs for details on each method. +*/ + +func (t *Target) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + return &delivery{ + t: t, + mailFrom: mailFrom, + log: DeliveryLogger(t.log, msgMeta), + msgMeta: msgMeta, + }, nil +} + +func (d *delivery) AddRcpt(ctx context.Context, rcptTo string) error { + // Corresponds to SMTP RCPT command. + panic("implement me") +} + +func (d *delivery) Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error { + // Corresponds to SMTP DATA command. + panic("implement me") +} + +/* +If Body call can fail partially (either success or fail for each recipient passed to AddRcpt) +- implement BodyNonAtomic and signal status for each recipient using StatusCollector callback. + +func (d *delivery) BodyNonAtomic(ctx context.Context, sc module.StatusCollector, header textproto.Header, body buffer.Buffer) { + +} +*/ + +func (d *delivery) Abort(ctx context.Context) error { + panic("implement me") +} + +func (d *delivery) Commit(ctx context.Context) error { + panic("implement me") +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/target/smtp/sasl.go b/internal/target/smtp/sasl.go new file mode 100644 index 0000000..75f5d4e --- /dev/null +++ b/internal/target/smtp/sasl.go @@ -0,0 +1,77 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp_downstream + +import ( + "github.com/emersion/go-sasl" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" +) + +type saslClientFactory = func(msgMeta *module.MsgMetadata) (sasl.Client, error) + +// saslAuthDirective returns saslClientFactory function used to create sasl.Client. +// for use in outbound connections. +// +// Authentication information of the current client should be passed in arguments. +func saslAuthDirective(_ *config.Map, node config.Node) (interface{}, error) { + if len(node.Children) != 0 { + return nil, config.NodeErr(node, "can't declare a block here") + } + if len(node.Args) == 0 { + return nil, config.NodeErr(node, "at least one argument required") + } + switch node.Args[0] { + case "off": + return nil, nil + case "forward": + if len(node.Args) > 1 { + return nil, config.NodeErr(node, "no additional arguments required") + } + return func(msgMeta *module.MsgMetadata) (sasl.Client, error) { + if msgMeta.Conn == nil || msgMeta.Conn.AuthUser == "" || msgMeta.Conn.AuthPassword == "" { + return nil, &exterrors.SMTPError{ + Code: 530, + EnhancedCode: exterrors.EnhancedCode{5, 7, 0}, + Message: "Authentication is required", + TargetName: "target.smtp", + Reason: "Credentials forwarding is requested but the client is not authenticated", + } + } + return sasl.NewPlainClient("", msgMeta.Conn.AuthUser, msgMeta.Conn.AuthPassword), nil + }, nil + case "plain": + if len(node.Args) != 3 { + return nil, config.NodeErr(node, "two additional arguments are required (username, password)") + } + return func(*module.MsgMetadata) (sasl.Client, error) { + return sasl.NewPlainClient("", node.Args[1], node.Args[2]), nil + }, nil + case "external": + if len(node.Args) > 1 { + return nil, config.NodeErr(node, "no additional arguments required") + } + return func(*module.MsgMetadata) (sasl.Client, error) { + return sasl.NewExternalClient(""), nil + }, nil + default: + return nil, config.NodeErr(node, "unknown authentication mechanism: %s", node.Args[0]) + } +} diff --git a/internal/target/smtp/sasl_test.go b/internal/target/smtp/sasl_test.go new file mode 100644 index 0000000..63b52a5 --- /dev/null +++ b/internal/target/smtp/sasl_test.go @@ -0,0 +1,154 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp_downstream + +import ( + "testing" + + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/testutils" +) + +func testSaslFactory(t *testing.T, args ...string) saslClientFactory { + factory, err := saslAuthDirective(&config.Map{}, config.Node{ + Name: "auth", + Args: args, + }) + if err != nil { + t.Fatal(err) + } + return factory.(saslClientFactory) +} + +func TestSASL_Plain(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + saslFactory: testSaslFactory(t, "plain", "test", "testpass"), + log: testutils.Logger(t, "target.smtp"), + } + + testutils.DoTestDelivery(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + be.CheckMsg(t, 0, "test@example.invalid", []string{"rcpt@example.invalid"}) + if be.Messages[0].AuthUser != "test" { + t.Errorf("Wrong AuthUser: %v", be.Messages[0].AuthUser) + } + if be.Messages[0].AuthPass != "testpass" { + t.Errorf("Wrong AuthPass: %v", be.Messages[0].AuthPass) + } +} + +func TestSASL_Plain_AuthFail(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + be.AuthErr = &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + } + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + saslFactory: testSaslFactory(t, "plain", "test", "testpass"), + log: testutils.Logger(t, "target.smtp"), + } + + _, err := testutils.DoTestDeliveryErr(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } +} + +func TestSASL_Forward(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + saslFactory: testSaslFactory(t, "forward"), + log: testutils.Logger(t, "target.smtp"), + } + + testutils.DoTestDeliveryMeta(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}, &module.MsgMetadata{ + Conn: &module.ConnState{ + AuthUser: "test", + AuthPassword: "testpass", + }, + }) + be.CheckMsg(t, 0, "test@example.invalid", []string{"rcpt@example.invalid"}) + if be.Messages[0].AuthUser != "test" { + t.Errorf("Wrong AuthUser: %v", be.Messages[0].AuthUser) + } + if be.Messages[0].AuthPass != "testpass" { + t.Errorf("Wrong AuthPass: %v", be.Messages[0].AuthPass) + } +} + +func TestSASL_Forward_NoCreds(t *testing.T) { + _, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + saslFactory: testSaslFactory(t, "forward"), + log: testutils.Logger(t, "target.smtp"), + } + + _, err := testutils.DoTestDeliveryErr(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } +} diff --git a/internal/target/smtp/smtp_downstream.go b/internal/target/smtp/smtp_downstream.go new file mode 100644 index 0000000..b85c4ba --- /dev/null +++ b/internal/target/smtp/smtp_downstream.go @@ -0,0 +1,336 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package smtp_downstream provides target.smtp module that implements +// transparent forwarding or messages to configured list of SMTP servers. +// +// Like remote module, this implementation doesn't handle atomic +// delivery properly since it is impossible to do with SMTP protocol +// +// Interfaces implemented: +// - module.DeliveryTarget +package smtp_downstream + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "runtime/trace" + "time" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + tls2 "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/smtpconn" + "github.com/foxcpp/maddy/internal/target" + "golang.org/x/net/idna" +) + +type Downstream struct { + modName string + instName string + lmtp bool + targetsArg []string + + starttls bool + hostname string + endpoints []config.Endpoint + saslFactory saslClientFactory + tlsConfig *tls.Config + + connectTimeout time.Duration + commandTimeout time.Duration + submissionTimeout time.Duration + + log log.Logger +} + +func (u *Downstream) moduleError(err error) error { + if err == nil { + return nil + } + + return exterrors.WithFields(err, map[string]interface{}{ + "target": u.modName, + }) +} + +func NewDownstream(modName, instName string, _, inlineArgs []string) (module.Module, error) { + return &Downstream{ + modName: modName, + instName: instName, + lmtp: modName == "target.lmtp" || modName == "lmtp_downstream", /* compatibility with 0.3 configs */ + targetsArg: inlineArgs, + log: log.Logger{Name: modName}, + }, nil +} + +func (u *Downstream) Init(cfg *config.Map) error { + var attemptTLS *bool + + var targetsArg []string + cfg.Bool("debug", true, false, &u.log.Debug) + cfg.Callback("require_tls", func(m *config.Map, node config.Node) error { + u.log.Msg("require_tls directive is deprecated and ignored") + return nil + }) + cfg.Callback("attempt_starttls", func(m *config.Map, node config.Node) error { + u.log.Msg("attempt_starttls directive is deprecated and equivalent to starttls") + + if len(node.Args) == 0 { + trueVal := true + attemptTLS = &trueVal + return nil + } + if len(node.Args) != 1 { + return config.NodeErr(node, "expected exactly 1 argument") + } + + b, err := config.ParseBool(node.Args[0]) + if err != nil { + return err + } + attemptTLS = &b + return nil + }) + cfg.Bool("starttls", false, !u.lmtp, &u.starttls) + cfg.String("hostname", true, true, "", &u.hostname) + cfg.StringList("targets", false, false, nil, &targetsArg) + cfg.Custom("auth", false, false, func() (interface{}, error) { + return nil, nil + }, saslAuthDirective, &u.saslFactory) + cfg.Custom("tls_client", true, false, func() (interface{}, error) { + return &tls.Config{}, nil + }, tls2.TLSClientBlock, &u.tlsConfig) + cfg.Duration("connect_timeout", false, false, 5*time.Minute, &u.connectTimeout) + cfg.Duration("command_timeout", false, false, 5*time.Minute, &u.commandTimeout) + cfg.Duration("submission_timeout", false, false, 5*time.Minute, &u.submissionTimeout) + + if _, err := cfg.Process(); err != nil { + return err + } + + if attemptTLS != nil { + u.starttls = *attemptTLS + } + + // INTERNATIONALIZATION: See RFC 6531 Section 3.7.1. + var err error + u.hostname, err = idna.ToASCII(u.hostname) + if err != nil { + return fmt.Errorf("%s: cannot represent the hostname as an A-label name: %w", u.modName, err) + } + + u.targetsArg = append(u.targetsArg, targetsArg...) + for _, tgt := range u.targetsArg { + endp, err := config.ParseEndpoint(tgt) + if err != nil { + return err + } + + u.endpoints = append(u.endpoints, endp) + } + + if len(u.endpoints) == 0 { + return fmt.Errorf("%s: at least one target endpoint is required", u.modName) + } + + return nil +} + +func (u *Downstream) Name() string { + return u.modName +} + +func (u *Downstream) InstanceName() string { + return u.instName +} + +type delivery struct { + u *Downstream + log log.Logger + + msgMeta *module.MsgMetadata + mailFrom string + rcpts []string + + conn *smtpconn.C +} + +// lmtpDelivery implements module.PartialDelivery +type lmtpDelivery struct { + *delivery +} + +func (u *Downstream) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + defer trace.StartRegion(ctx, "target.smtp/Start").End() + + d := &delivery{ + u: u, + log: target.DeliveryLogger(u.log, msgMeta), + msgMeta: msgMeta, + mailFrom: mailFrom, + } + if err := d.connect(ctx); err != nil { + return nil, err + } + + if err := d.conn.Mail(ctx, mailFrom, msgMeta.SMTPOpts); err != nil { + d.conn.Close() + return nil, err + } + + if u.lmtp { + return &lmtpDelivery{delivery: d}, nil + } + + return d, nil +} + +func (d *delivery) connect(ctx context.Context) error { + // TODO: Review possibility of connection pooling here. + var lastErr error + + conn := smtpconn.New() + conn.Log = d.log + conn.Hostname = d.u.hostname + conn.AddrInSMTPMsg = false + if d.u.connectTimeout != 0 { + conn.ConnectTimeout = d.u.connectTimeout + } + if d.u.commandTimeout != 0 { + conn.CommandTimeout = d.u.commandTimeout + } + if d.u.submissionTimeout != 0 { + conn.SubmissionTimeout = d.u.submissionTimeout + } + + for _, endp := range d.u.endpoints { + var err error + if d.u.lmtp { + _, err = conn.ConnectLMTP(ctx, endp, d.u.starttls, d.u.tlsConfig) + } else { + _, err = conn.Connect(ctx, endp, d.u.starttls, d.u.tlsConfig) + } + if err != nil { + if len(d.u.endpoints) != 1 { + d.log.Msg("connect error", err, "downstream_server", net.JoinHostPort(endp.Host, endp.Port)) + } + lastErr = err + continue + } + + d.log.DebugMsg("connected", "downstream_server", conn.ServerName()) + + lastErr = nil + break + } + if lastErr != nil { + return d.u.moduleError(lastErr) + } + + if d.u.saslFactory != nil { + saslClient, err := d.u.saslFactory(d.msgMeta) + if err != nil { + conn.Close() + return err + } + + if err := conn.Client().Auth(saslClient); err != nil { + conn.Close() + return err + } + } + + d.conn = conn + + return nil +} + +func (d *delivery) AddRcpt(ctx context.Context, rcptTo string, opts smtp.RcptOptions) error { + err := d.conn.Rcpt(ctx, rcptTo, opts) + if err != nil { + return d.u.moduleError(err) + } + + d.rcpts = append(d.rcpts, rcptTo) + return nil +} + +func (d *delivery) Body(ctx context.Context, header textproto.Header, body buffer.Buffer) error { + r, err := body.Open() + if err != nil { + return exterrors.WithFields(err, map[string]interface{}{"target": d.u.modName}) + } + + defer r.Close() + return d.u.moduleError(d.conn.Data(ctx, header, r)) +} + +func (d *lmtpDelivery) BodyNonAtomic(ctx context.Context, sc module.StatusCollector, header textproto.Header, body buffer.Buffer) { + r, err := body.Open() + if err != nil { + modErr := d.u.moduleError(err) + for _, rcpt := range d.rcpts { + sc.SetStatus(rcpt, modErr) + } + } + defer r.Close() + + rcptIndx := 0 + err = d.conn.LMTPData(ctx, header, r, func(rcpt string, err *smtp.SMTPError) { + if err == nil { + sc.SetStatus(rcpt, nil) + } else { + sc.SetStatus(rcpt, &exterrors.SMTPError{ + Code: err.Code, + EnhancedCode: exterrors.EnhancedCode(err.EnhancedCode), + Message: err.Message, + TargetName: d.u.modName, + Err: err, + }) + } + rcptIndx++ + }) + if err != nil { + modErr := d.u.moduleError(err) + for _, rcpt := range d.rcpts[rcptIndx:] { + sc.SetStatus(rcpt, modErr) + } + } +} + +func (d *delivery) Abort(ctx context.Context) error { + d.conn.Close() + return nil +} + +func (d *delivery) Commit(ctx context.Context) error { + return d.conn.Close() +} + +func init() { + module.Register("target.smtp", NewDownstream) + module.Register("target.lmtp", NewDownstream) +} diff --git a/internal/target/smtp/smtp_downstream_test.go b/internal/target/smtp/smtp_downstream_test.go new file mode 100644 index 0000000..f0f58c6 --- /dev/null +++ b/internal/target/smtp/smtp_downstream_test.go @@ -0,0 +1,272 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp_downstream + +import ( + "errors" + "flag" + "math/rand" + "os" + "strconv" + "testing" + + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/internal/testutils" +) + +var testPort string + +func TestDownstreamDelivery(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + tarpit := testutils.FailOnConn(t, "127.0.0.2:"+testPort) + defer tarpit.Close() + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + { + Scheme: "tcp", + Host: "127.0.0.2", + Port: testPort, + }, + }, + log: testutils.Logger(t, "target.smtp"), + } + + testutils.DoTestDelivery(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + be.CheckMsg(t, 0, "test@example.invalid", []string{"rcpt@example.invalid"}) +} + +func TestDownstreamDelivery_LMTP(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort, func(srv *smtp.Server) { + srv.LMTP = true + }) + be.LMTPDataErr = []error{ + nil, + &smtp.SMTPError{ + Code: 501, + Message: "nop", + }, + } + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + modName: "target.lmtp", + lmtp: true, + log: testutils.Logger(t, "lmtp_downstream"), + } + + sc := make(statusCollector) + + testutils.DoTestDeliveryNonAtomic(t, &sc, mod, "test@example.invalid", []string{"rcpt1@example.invalid", "rcpt2@example.invalid"}) + be.CheckMsg(t, 0, "test@example.invalid", []string{"rcpt1@example.invalid", "rcpt2@example.invalid"}) + + if len(sc) != 2 { + t.Fatal("Two statuses should be set") + } + if err := sc["rcpt1@example.invalid"]; err != nil { + t.Fatal("Unexpected error for rcpt1:", err) + } + if sc["rcpt2@example.invalid"] == nil { + t.Fatal("Expected an error for rcpt2") + } + var rcptErr *exterrors.SMTPError + if !errors.As(sc["rcpt2@example.invalid"], &rcptErr) { + t.Fatalf("Not SMTPError: %T", rcptErr) + } + if rcptErr.Code != 501 { + t.Fatal("Wrong SMTP code:", rcptErr.Code) + } +} + +func TestDownstreamDelivery_LMTP_ErrorCoerce(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort, func(srv *smtp.Server) { + srv.LMTP = true + }) + be.LMTPDataErr = []error{ + nil, + &smtp.SMTPError{ + Code: 501, + Message: "nop", + }, + } + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + modName: "target.lmtp", + lmtp: true, + log: testutils.Logger(t, "lmtp_downstream"), + } + + _, err := testutils.DoTestDeliveryErr(t, mod, "test@example.invalid", []string{"rcpt1@example.invalid", "rcpt2@example.invalid"}) + if err == nil { + t.Error("expected failure") + } +} + +type statusCollector map[string]error + +func (sc *statusCollector) SetStatus(rcptTo string, err error) { + (*sc)[rcptTo] = err +} + +func TestDownstreamDelivery_Fallback(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.2:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + { + Scheme: "tcp", + Host: "127.0.0.2", + Port: testPort, + }, + }, + log: testutils.Logger(t, "target.smtp"), + } + + testutils.DoTestDelivery(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + be.CheckMsg(t, 0, "test@example.invalid", []string{"rcpt@example.invalid"}) +} + +func TestDownstreamDelivery_MAILErr(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + be.MailErr = &smtp.SMTPError{ + Code: 550, + EnhancedCode: smtp.EnhancedCode{5, 1, 2}, + Message: "Hey", + } + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + log: testutils.Logger(t, "target.smtp"), + } + + _, err := testutils.DoTestDeliveryErr(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + testutils.CheckSMTPErr(t, err, 550, exterrors.EnhancedCode{5, 1, 2}, "Hey") +} + +func TestDownstreamDelivery_StartTLS(t *testing.T) { + clientCfg, be, srv := testutils.SMTPServerSTARTTLS(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + tlsConfig: clientCfg.Clone(), + starttls: true, + log: testutils.Logger(t, "target.smtp"), + } + + testutils.DoTestDelivery(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + be.CheckMsg(t, 0, "test@example.invalid", []string{"rcpt@example.invalid"}) + + tlsState, ok := be.Messages[0].Conn.TLSConnectionState() + if !ok || !tlsState.HandshakeComplete { + t.Fatal("Message was not delivered over TLS") + } +} + +func TestDownstreamDelivery_StartTLS_NoFallback(t *testing.T) { + _, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod := &Downstream{ + hostname: "mx.example.invalid", + endpoints: []config.Endpoint{ + { + Scheme: "tcp", + Host: "127.0.0.1", + Port: testPort, + }, + }, + starttls: true, + log: testutils.Logger(t, "target.smtp"), + } + + _, err := testutils.DoTestDeliveryErr(t, mod, "test@example.invalid", []string{"rcpt@example.invalid"}) + if err == nil { + t.Error("Expected an error, got none") + } +} + +func TestMain(m *testing.M) { + remoteSmtpPort := flag.String("test.smtpport", "random", "(maddy) SMTP port to use for connections in tests") + flag.Parse() + + if *remoteSmtpPort == "random" { + *remoteSmtpPort = strconv.Itoa(rand.Intn(65536-10000) + 10000) + } + + testPort = *remoteSmtpPort + os.Exit(m.Run()) +} diff --git a/internal/target/smtp/smtputf8_test.go b/internal/target/smtp/smtputf8_test.go new file mode 100644 index 0000000..74aae23 --- /dev/null +++ b/internal/target/smtp/smtputf8_test.go @@ -0,0 +1,61 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package smtp_downstream + +import ( + "testing" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/internal/testutils" +) + +func TestDownstreamDelivery_EHLO_ALabel(t *testing.T) { + be, srv := testutils.SMTPServer(t, "127.0.0.1:"+testPort) + defer srv.Close() + defer testutils.CheckSMTPConnLeak(t, srv) + + mod, err := NewDownstream("", "", nil, []string{"tcp://127.0.0.1:" + testPort}) + if err != nil { + t.Fatal(err) + } + if err := mod.Init(config.NewMap(nil, config.Node{ + Children: []config.Node{ + { + Name: "hostname", + Args: []string{"тест.invalid"}, + }, + { + Name: "starttls", + Args: []string{"no"}, + }, + }, + })); err != nil { + t.Fatal(err) + } + + tgt := mod.(*Downstream) + tgt.log = testutils.Logger(t, "remote") + + testutils.DoTestDelivery(t, tgt, "test@example.com", []string{"test@example.invalid"}) + + be.CheckMsg(t, 0, "test@example.com", []string{"test@example.invalid"}) + if be.Messages[0].Conn.Hostname() != "xn--e1aybc.invalid" { + t.Error("target/remote should use use Punycode in EHLO") + } +} diff --git a/internal/testutils/bench_delivery.go b/internal/testutils/bench_delivery.go new file mode 100644 index 0000000..f434efc --- /dev/null +++ b/internal/testutils/bench_delivery.go @@ -0,0 +1,141 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "bufio" + "context" + "crypto/sha1" + "encoding/hex" + "io" + "strconv" + "strings" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/module" +) + +// Empirically observed "around average" values. +const ( + MessageBodySize = 100 * 1024 + ExtraMessageHeaderFields = 10 + ExtraMessageHeaderFieldSize = 50 +) + +const testHeaderString = "Content-Type: multipart/mixed; boundary=message-boundary\r\n" + + "Date: Sat, 19 Jun 2016 12:00:00 +0900\r\n" + + "From: Mitsuha Miyamizu \r\n" + + "Reply-To: Mitsuha Miyamizu \r\n" + + "Message-Id: 42@example.org\r\n" + + "MIME-Version: 1.0\r\n" + + "Content-Transfer-Encoding: 8but\r\n" + + "Subject: Your Name.\r\n" + + "To: Taki Tachibana \r\n" + + "\r\n" + +const testAltHeaderString = "Content-Type: multipart/alternative; boundary=b2\r\n" + + "\r\n" + +const testTextHeaderString = "Content-Disposition: inline\r\n" + + "Content-Type: text/plain\r\n" + + "\r\n" + +const testTextBodyString = "What's your name?" + +const testTextString = testTextHeaderString + testTextBodyString + +const testHTMLHeaderString = "Content-Disposition: inline\r\n" + + "Content-Type: text/html\r\n" + + "\r\n" + +const testHTMLBodyString = "
What's your name?
" + +const testHTMLString = testHTMLHeaderString + testHTMLBodyString + +const testAttachmentHeaderString = "Content-Disposition: attachment; filename=note.txt\r\n" + + "Content-Type: text/plain\r\n" + + "\r\n" + +const testAttachmentBodyString = "My name is Mitsuha." + +const testAttachmentString = testAttachmentHeaderString + testAttachmentBodyString + +const testBodyString = "--message-boundary\r\n" + + testAltHeaderString + + "\r\n--b2\r\n" + + testTextString + + "\r\n--b2\r\n" + + testHTMLString + + "\r\n--b2--\r\n" + + "\r\n--message-boundary\r\n" + + testAttachmentString + + "\r\n--message-boundary--\r\n" + +var testMailString = testHeaderString + testBodyString + strings.Repeat("A", MessageBodySize) + +func RandomMsg(b *testing.B) (module.MsgMetadata, textproto.Header, buffer.Buffer) { + IDRaw := sha1.Sum([]byte(b.Name())) + encodedID := hex.EncodeToString(IDRaw[:]) + + body := bufio.NewReader(strings.NewReader(testMailString)) + hdr, _ := textproto.ReadHeader(body) + for i := 0; i < ExtraMessageHeaderFields; i++ { + hdr.Add("AAAAAAAAAAAA-"+strconv.Itoa(i), strings.Repeat("A", ExtraMessageHeaderFieldSize)) + } + bodyBlob, _ := io.ReadAll(body) + + return module.MsgMetadata{ + DontTraceSender: true, + ID: encodedID, + }, hdr, buffer.MemoryBuffer{Slice: bodyBlob} +} + +func BenchDelivery(b *testing.B, target module.DeliveryTarget, sender string, recipientTemplates []string) { + meta, header, body := RandomMsg(b) + + benchCtx := context.Background() + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + delivery, err := target.Start(benchCtx, &meta, sender) + if err != nil { + b.Fatal(err) + } + + for i, rcptTemplate := range recipientTemplates { + rcpt := strings.Replace(rcptTemplate, "X", strconv.Itoa(i), -1) + + if err := delivery.AddRcpt(benchCtx, rcpt, smtp.RcptOptions{}); err != nil { + b.Fatal(err) + } + } + + if err := delivery.Body(benchCtx, header, body); err != nil { + b.Fatal(err) + } + + if err := delivery.Commit(benchCtx); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/testutils/buffer.go b/internal/testutils/buffer.go new file mode 100644 index 0000000..259eea2 --- /dev/null +++ b/internal/testutils/buffer.go @@ -0,0 +1,84 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "bufio" + "bytes" + "io" + "strings" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" +) + +func BodyFromStr(t *testing.T, literal string) (textproto.Header, buffer.MemoryBuffer) { + t.Helper() + + bufr := bufio.NewReader(strings.NewReader(literal)) + hdr, err := textproto.ReadHeader(bufr) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(bufr) + if err != nil { + t.Fatal(err) + } + + return hdr, buffer.MemoryBuffer{Slice: body} +} + +type errorReader struct { + r io.Reader + err error +} + +func (r *errorReader) Read(b []byte) (int, error) { + n, err := r.r.Read(b) + if err == io.EOF { + return n, r.err + } + return n, err +} + +type FailingBuffer struct { + Blob []byte + + OpenError error + IOError error +} + +func (fb FailingBuffer) Open() (io.ReadCloser, error) { + r := io.NopCloser(bytes.NewReader(fb.Blob)) + + if fb.IOError != nil { + return io.NopCloser(&errorReader{r, fb.IOError}), fb.OpenError + } + + return r, fb.OpenError +} + +func (fb FailingBuffer) Len() int { + return len(fb.Blob) +} + +func (fb FailingBuffer) Remove() error { + return nil +} diff --git a/internal/testutils/check.go b/internal/testutils/check.go new file mode 100644 index 0000000..399a78d --- /dev/null +++ b/internal/testutils/check.go @@ -0,0 +1,111 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +type Check struct { + InitErr error + EarlyErr error + ConnRes module.CheckResult + SenderRes module.CheckResult + RcptRes module.CheckResult + BodyRes module.CheckResult + + ConnCalls int + SenderCalls int + RcptCalls int + BodyCalls int + + UnclosedStates int + + InstName string +} + +func (c *Check) CheckStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.CheckState, error) { + if c.InitErr != nil { + return nil, c.InitErr + } + + c.UnclosedStates++ + return &checkState{msgMeta, c}, nil +} + +func (c *Check) Init(*config.Map) error { + return nil +} + +func (c *Check) Name() string { + return "test_check" +} + +func (c *Check) InstanceName() string { + if c.InstName != "" { + return c.InstName + } + return "test_check" +} + +func (c *Check) CheckConnection(ctx context.Context, state *module.ConnState) error { + return c.EarlyErr +} + +type checkState struct { + msgMeta *module.MsgMetadata + check *Check +} + +func (cs *checkState) CheckConnection(ctx context.Context) module.CheckResult { + cs.check.ConnCalls++ + return cs.check.ConnRes +} + +func (cs *checkState) CheckSender(ctx context.Context, from string) module.CheckResult { + cs.check.SenderCalls++ + return cs.check.SenderRes +} + +func (cs *checkState) CheckRcpt(ctx context.Context, to string) module.CheckResult { + cs.check.RcptCalls++ + return cs.check.RcptRes +} + +func (cs *checkState) CheckBody(ctx context.Context, header textproto.Header, body buffer.Buffer) module.CheckResult { + cs.check.BodyCalls++ + return cs.check.BodyRes +} + +func (cs *checkState) Close() error { + cs.check.UnclosedStates-- + return nil +} + +func init() { + module.Register("test_check", func(_, _ string, _, _ []string) (module.Module, error) { + return &Check{}, nil + }) + module.RegisterInstance(&Check{}, nil) +} diff --git a/internal/testutils/filesystem.go b/internal/testutils/filesystem.go new file mode 100644 index 0000000..cdac017 --- /dev/null +++ b/internal/testutils/filesystem.go @@ -0,0 +1,34 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "os" + "testing" +) + +// Dir is a wrapper for os.MkdirTemp that +// fails the test on errors. +func Dir(t *testing.T) string { + dir, err := os.MkdirTemp("", "maddy-tests-") + if err != nil { + t.Fatalf("can't create test dir: %v", err) + } + return dir +} diff --git a/internal/testutils/logger.go b/internal/testutils/logger.go new file mode 100644 index 0000000..9fd5506 --- /dev/null +++ b/internal/testutils/logger.go @@ -0,0 +1,59 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "flag" + "os" + "strings" + "testing" + "time" + + "github.com/foxcpp/maddy/framework/log" +) + +var ( + debugLog = flag.Bool("test.debuglog", false, "(maddy) Turn on debug log messages") + directLog = flag.Bool("test.directlog", false, "(maddy) Log to stderr instead of test log") +) + +func Logger(t *testing.T, name string) log.Logger { + if *directLog { + return log.Logger{ + Out: log.WriterOutput(os.Stderr, true), + Name: name, + Debug: *debugLog, + } + } + + return log.Logger{ + Out: log.FuncOutput(func(_ time.Time, debug bool, str string) { + t.Helper() + str = strings.TrimSuffix(str, "\n") + if debug { + str = "[debug] " + str + } + t.Log(str) + }, func() error { + return nil + }), + Name: name, + Debug: *debugLog, + } +} diff --git a/internal/testutils/modifier.go b/internal/testutils/modifier.go new file mode 100644 index 0000000..a96cbe4 --- /dev/null +++ b/internal/testutils/modifier.go @@ -0,0 +1,122 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "context" + + "github.com/emersion/go-message/textproto" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +type Modifier struct { + InstName string + + InitErr error + MailFromErr error + RcptToErr error + BodyErr error + + MailFrom map[string]string + RcptTo map[string][]string + AddHdr textproto.Header + + UnclosedStates int +} + +func (m Modifier) Init(*config.Map) error { + return nil +} + +func (m Modifier) Name() string { + return "test_modifier" +} + +func (m Modifier) InstanceName() string { + return m.InstName +} + +type modifierState struct { + m *Modifier +} + +func (m Modifier) ModStateForMsg(ctx context.Context, msgMeta *module.MsgMetadata) (module.ModifierState, error) { + if m.InitErr != nil { + return nil, m.InitErr + } + + m.UnclosedStates++ + return modifierState{&m}, nil +} + +func (ms modifierState) RewriteSender(ctx context.Context, mailFrom string) (string, error) { + if ms.m.MailFromErr != nil { + return "", ms.m.MailFromErr + } + if ms.m.MailFrom == nil { + return mailFrom, nil + } + + newMailFrom, ok := ms.m.MailFrom[mailFrom] + if ok { + return newMailFrom, nil + } + return mailFrom, nil +} + +func (ms modifierState) RewriteRcpt(ctx context.Context, rcptTo string) ([]string, error) { + if ms.m.RcptToErr != nil { + return []string{""}, ms.m.RcptToErr + } + + if ms.m.RcptTo == nil { + return []string{rcptTo}, nil + } + + newRcptTo, ok := ms.m.RcptTo[rcptTo] + if ok { + return newRcptTo, nil + } + return []string{rcptTo}, nil +} + +func (ms modifierState) RewriteBody(ctx context.Context, h *textproto.Header, body buffer.Buffer) error { + if ms.m.BodyErr != nil { + return ms.m.BodyErr + } + + for field := ms.m.AddHdr.Fields(); field.Next(); { + h.Add(field.Key(), field.Value()) + } + return nil +} + +func (ms modifierState) Close() error { + ms.m.UnclosedStates-- + return nil +} + +func init() { + module.Register("test_modifier", func(_, _ string, _, _ []string) (module.Module, error) { + return &Modifier{}, nil + }) + module.RegisterInstance(&Modifier{}, nil) +} diff --git a/internal/testutils/multitable.go b/internal/testutils/multitable.go new file mode 100644 index 0000000..9b84abe --- /dev/null +++ b/internal/testutils/multitable.go @@ -0,0 +1,35 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import "context" + +type MultiTable struct { + M map[string][]string + Err error +} + +func (m MultiTable) LookupMulti(_ context.Context, a string) ([]string, error) { + b, ok := m.M[a] + if ok { + return b, m.Err + } else { + return []string{}, m.Err + } +} diff --git a/internal/testutils/smtp_server.go b/internal/testutils/smtp_server.go new file mode 100644 index 0000000..9af52c4 --- /dev/null +++ b/internal/testutils/smtp_server.go @@ -0,0 +1,454 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net" + "reflect" + "sort" + "sync/atomic" + "testing" + "time" + + "github.com/emersion/go-sasl" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/exterrors" +) + +type SMTPMessage struct { + From string + Opts smtp.MailOptions + To []string + Data []byte + Conn *smtp.Conn + AuthUser string + AuthPass string +} + +type SMTPBackend struct { + Messages []*SMTPMessage + MailFromCounter int + SessionCounter int + SourceEndpoints map[string]struct{} + + AuthErr error + MailErr error + RcptErr map[string]error + DataErr error + LMTPDataErr []error + + ActiveSessionsCounter atomic.Int32 +} + +func (be *SMTPBackend) NewSession(conn *smtp.Conn) (smtp.Session, error) { + be.SessionCounter++ + be.ActiveSessionsCounter.Add(1) + if be.SourceEndpoints == nil { + be.SourceEndpoints = make(map[string]struct{}) + } + be.SourceEndpoints[conn.Conn().RemoteAddr().String()] = struct{}{} + return &session{ + backend: be, + conn: conn, + }, nil +} + +func (be *SMTPBackend) ConnectionCount() int { + return int(be.ActiveSessionsCounter.Load()) +} + +func (be *SMTPBackend) CheckMsg(t *testing.T, indx int, from string, rcptTo []string) { + t.Helper() + + if len(be.Messages) <= indx { + t.Errorf("Expected at least %d messages in mailbox, got %d", indx+1, len(be.Messages)) + return + } + + msg := be.Messages[indx] + if msg.From != from { + t.Errorf("Wrong MAIL FROM: %v", msg.From) + } + + sort.Strings(msg.To) + sort.Strings(rcptTo) + + if !reflect.DeepEqual(msg.To, rcptTo) { + t.Errorf("Wrong RCPT TO: %v", msg.To) + } + if string(msg.Data) != DeliveryData { + t.Errorf("Wrong DATA payload: %v (%v)", string(msg.Data), msg.Data) + } +} + +type session struct { + backend *SMTPBackend + user string + password string + conn *smtp.Conn + msg *SMTPMessage +} + +func (s *session) AuthMechanisms() []string { + return []string{sasl.Plain} +} + +func (s *session) Auth(mech string) (sasl.Server, error) { + if mech != sasl.Plain { + return nil, fmt.Errorf("mechanisms other than plain are unsupported") + } + return sasl.NewPlainServer(func(identity, username, password string) error { + if s.backend.AuthErr != nil { + return s.backend.AuthErr + } + s.user = username + s.password = password + return nil + }), nil +} + +func (s *session) Reset() { + s.msg = &SMTPMessage{} +} + +func (s *session) Logout() error { + s.backend.ActiveSessionsCounter.Add(-1) + return nil +} + +func (s *session) Mail(from string, opts *smtp.MailOptions) error { + s.backend.MailFromCounter++ + + if s.backend.MailErr != nil { + return s.backend.MailErr + } + + s.Reset() + s.msg.From = from + s.msg.Opts = *opts + return nil +} + +func (s *session) Rcpt(to string, _ *smtp.RcptOptions) error { + if err := s.backend.RcptErr[to]; err != nil { + return err + } + + s.msg.To = append(s.msg.To, to) + return nil +} + +func (s *session) Data(r io.Reader) error { + if s.backend.DataErr != nil { + return s.backend.DataErr + } + + b, err := io.ReadAll(r) + if err != nil { + return err + } + s.msg.Data = b + s.msg.Conn = s.conn + s.msg.AuthUser = s.user + s.msg.AuthPass = s.password + s.backend.Messages = append(s.backend.Messages, s.msg) + return nil +} + +func (s *session) LMTPData(r io.Reader, status smtp.StatusCollector) error { + if s.backend.DataErr != nil { + return s.backend.DataErr + } + + b, err := io.ReadAll(r) + if err != nil { + return err + } + s.msg.Data = b + s.msg.Conn = s.conn + s.msg.AuthUser = s.user + s.msg.AuthPass = s.password + s.backend.Messages = append(s.backend.Messages, s.msg) + + for i, rcpt := range s.msg.To { + status.SetStatus(rcpt, s.backend.LMTPDataErr[i]) + } + + return nil +} + +type SMTPServerConfigureFunc func(*smtp.Server) + +func SMTPServer(t *testing.T, addr string, fn ...SMTPServerConfigureFunc) (*SMTPBackend, *smtp.Server) { + t.Helper() + + l, err := net.Listen("tcp", addr) + if err != nil { + t.Fatal(err) + } + + be := new(SMTPBackend) + s := smtp.NewServer(be) + s.Domain = "localhost" + s.AllowInsecureAuth = true + for _, f := range fn { + f(s) + } + + go func() { + if err := s.Serve(l); err != nil { + t.Error(err) + } + }() + + // Dial it once it make sure Server completes its initialization before + // we try to use it. Notably, if test fails before connecting to the server, + // it will call Server.Close which will call Server.listener.Close with a + // nil Server.listener (Serve sets it to a non-nil value, so it is racy and + // happens only sometimes). + testConn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + testConn.Close() + + return be, s +} + +// RSA 1024, valid for *.example.invalid, 127.0.0.1, 127.0.0.2,, 127.0.0.3 +// until Nov 18 17:13:45 2029 GMT. +const testServerCert = `-----BEGIN CERTIFICATE----- +MIICDzCCAXigAwIBAgIRAJ1x+qCW7L+Hs6sRU8BHmWkwDQYJKoZIhvcNAQELBQAw +EjEQMA4GA1UEChMHQWNtZSBDbzAeFw0xOTExMTgxNzEzNDVaFw0yOTExMTUxNzEz +NDVaMBIxEDAOBgNVBAoTB0FjbWUgQ28wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ +AoGBAPINKMyuu3AvzndLDS2/BroA+DRUcAhWPBxMxG1b1BkkHisAZWteKajKmwdO +O13N8HHBRPPOD56AAPLZGNxYLHn6nel7AiH8k40/xC5tDOthqA82+00fwJHDFCnW +oDLOLcO17HulPvfCSWfefc+uee4kajPa+47hutqZH2bGMTXhAgMBAAGjZTBjMA4G +A1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAA +MC4GA1UdEQQnMCWCESouZXhhbXBsZS5pbnZhbGlkhwR/AAABhwR/AAAChwR/AAAD +MA0GCSqGSIb3DQEBCwUAA4GBAGRn3C2NbwR4cyQmTRm5jcaqi1kAYyEu6U8Q9PJW +Q15BXMKUTx2lw//QScK9MH2JpKxDuzWDSvaxZMnTxgri2uiplqpe8ydsWj6Wl0q9 +2XMGJ9LIxTZk5+cyZP2uOolvmSP/q8VFTyk9Udl6KUZPQyoiiDq4rBFUIxUyb+bX +pHkR +-----END CERTIFICATE-----` + +const testServerKey = `-----BEGIN PRIVATE KEY----- +MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAPINKMyuu3AvzndL +DS2/BroA+DRUcAhWPBxMxG1b1BkkHisAZWteKajKmwdOO13N8HHBRPPOD56AAPLZ +GNxYLHn6nel7AiH8k40/xC5tDOthqA82+00fwJHDFCnWoDLOLcO17HulPvfCSWfe +fc+uee4kajPa+47hutqZH2bGMTXhAgMBAAECgYEAgPjSDH3uEdDnSlkLJJzskJ+D +oR58s3R/gvTElSCg2uSLzo3ffF4oBHAwOqxMpabdvz8j5mSdne7Gkp9qx72TtEG2 +wt6uX1tZhm2UTAkInH8IQDthj98P8vAWQsS6HHEIMErsrW2CyUrAt/+o1BRg/hWW +zixA3CLTthhZTJkaUCECQQD5EM16UcTAKfhr3IZppgq+ZsAOMkeCl3XVV9gHo32i +DL6UFAb27BAYyjfcZB1fPou4RszX0Ryu9yU0P5qm6N47AkEA+MpdAPkaPziY0ok4 +e9Tcee6P0mIR+/AHk9GliVX2P74DDoOHyMXOSRBwdb+z2tYjrdjkNEL1Txe+sHny +k/EukwJBAOBqlmqPwNNRPeiaRHZvSSD0XjqsbSirJl48D4gadPoNt66fOQNGAt8D +Xj/z6U9HgQdiq/IOFmVEhT5FzSh1jL8CQQD3Myth8iGQO84tM0c6U3CWfuHMqsEv +0XnV+HNAmHdLMqOa4joi1dh4ZKs5dDdi828UJ/PnsbhI1FEWzLSpJvWdAkAkVWqf +AC/TvWvEZLA6Z5CllyNzZJ7XvtIaNOosxHDolyZ1HMWMlfEb2K2ZXWLy5foKPeoY +Xi3olS9rB0J+Rvjz +-----END PRIVATE KEY-----` + +// SMTPServerSTARTTLS starts a server listening on the specified addr with the +// STARTTLS extension supported. +// +// Returned *tls.Config is for the client and is set to trust the server +// certificate. +func SMTPServerSTARTTLS(t *testing.T, addr string, fn ...SMTPServerConfigureFunc) (*tls.Config, *SMTPBackend, *smtp.Server) { + t.Helper() + + cert, err := tls.X509KeyPair([]byte(testServerCert), []byte(testServerKey)) + if err != nil { + panic(err) + } + + l, err := net.Listen("tcp", addr) + if err != nil { + t.Fatal(err) + } + + be := new(SMTPBackend) + s := smtp.NewServer(be) + s.Domain = "localhost" + s.AllowInsecureAuth = true + s.TLSConfig = &tls.Config{ + Certificates: []tls.Certificate{cert}, + } + for _, f := range fn { + f(s) + } + + pool := x509.NewCertPool() + pool.AppendCertsFromPEM([]byte(testServerCert)) + + clientCfg := &tls.Config{ + ServerName: "127.0.0.1", + Time: func() time.Time { + return time.Date(2019, time.November, 18, 17, 59, 41, 0, time.UTC) + }, + RootCAs: pool, + } + + go func() { + if err := s.Serve(l); err != nil { + t.Error(err) + } + }() + + // Dial it once it make sure Server completes its initialization before + // we try to use it. Notably, if test fails before connecting to the server, + // it will call Server.Close which will call Server.listener.Close with a + // nil Server.listener (Serve sets it to a non-nil value, so it is racy and + // happens only sometimes). + testConn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + testConn.Close() + + return clientCfg, be, s +} + +// SMTPServerTLS starts a SMTP server listening on the specified addr with +// Implicit TLS. +func SMTPServerTLS(t *testing.T, addr string, fn ...SMTPServerConfigureFunc) (*tls.Config, *SMTPBackend, *smtp.Server) { + t.Helper() + + cert, err := tls.X509KeyPair([]byte(testServerCert), []byte(testServerKey)) + if err != nil { + panic(err) + } + + l, err := tls.Listen("tcp", addr, &tls.Config{ + Certificates: []tls.Certificate{cert}, + }) + if err != nil { + t.Fatal(err) + } + + be := new(SMTPBackend) + s := smtp.NewServer(be) + s.Domain = "localhost" + for _, f := range fn { + f(s) + } + + pool := x509.NewCertPool() + pool.AppendCertsFromPEM([]byte(testServerCert)) + + clientCfg := &tls.Config{ + ServerName: "127.0.0.1", + Time: func() time.Time { + return time.Date(2019, time.November, 18, 17, 59, 41, 0, time.UTC) + }, + RootCAs: pool, + } + + go func() { + if err := s.Serve(l); err != nil { + t.Error(err) + } + }() + + // Dial it once it make sure Server completes its initialization before + // we try to use it. Notably, if test fails before connecting to the server, + // it will call Server.Close which will call Server.listener.Close with a + // nil Server.listener (Serve sets it to a non-nil value, so it is racy and + // happens only sometimes). + testConn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + testConn.Close() + + return clientCfg, be, s +} + +type smtpBackendConnCounter interface { + ConnectionCount() int +} + +func CheckSMTPConnLeak(t *testing.T, srv *smtp.Server) { + t.Helper() + + ccb, ok := srv.Backend.(smtpBackendConnCounter) + if !ok { + t.Error("CheckSMTPConnLeak used for smtp.Server with backend without ConnectionCount method") + return + } + + // Connection closure is handled asynchronously, so before failing + // wait a bit for handleQuit in go-smtp to do its work. + for i := 0; i < 10; i++ { + if ccb.ConnectionCount() == 0 { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Error("Non-closed connections present after test completion") +} + +func WaitForConnsClose(t *testing.T, srv *smtp.Server) { + t.Helper() + CheckSMTPConnLeak(t, srv) +} + +// FailOnConn fails the test if attempt is made to connect the +// specified endpoint. +func FailOnConn(t *testing.T, addr string) net.Listener { + t.Helper() + + tarpit, err := net.Listen("tcp", addr) + if err != nil { + t.Fatal(err) + } + go func() { + t.Helper() + + _, err := tarpit.Accept() + if err == nil { + t.Error("No connection expected") + } + }() + return tarpit +} + +func CheckSMTPErr(t *testing.T, err error, code int, enchCode exterrors.EnhancedCode, msg string) { + t.Helper() + + if err == nil { + t.Error("Expected an error, got none") + return + } + + fields := exterrors.Fields(err) + if val, _ := fields["smtp_code"].(int); val != code { + t.Errorf("Wrong smtp_code: %v", val) + } + if val, _ := fields["smtp_enchcode"].(exterrors.EnhancedCode); val != enchCode { + t.Errorf("Wrong smtp_enchcode: %v", val) + } + if val, _ := fields["smtp_msg"].(string); val != msg { + t.Errorf("Wrong smtp_msg: %v", val) + } +} diff --git a/internal/testutils/table.go b/internal/testutils/table.go new file mode 100644 index 0000000..ec108a8 --- /dev/null +++ b/internal/testutils/table.go @@ -0,0 +1,31 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import "context" + +type Table struct { + M map[string]string + Err error +} + +func (m Table) Lookup(_ context.Context, a string) (string, bool, error) { + b, ok := m.M[a] + return b, ok, m.Err +} diff --git a/internal/testutils/target.go b/internal/testutils/target.go new file mode 100644 index 0000000..68f5b39 --- /dev/null +++ b/internal/testutils/target.go @@ -0,0 +1,344 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package testutils + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "io" + "reflect" + "sort" + "testing" + + "github.com/emersion/go-message/textproto" + "github.com/emersion/go-smtp" + "github.com/foxcpp/maddy/framework/buffer" + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/exterrors" + "github.com/foxcpp/maddy/framework/module" +) + +type Msg struct { + MsgMeta *module.MsgMetadata + MailFrom string + RcptTo []string + Body []byte + Header textproto.Header +} + +type Target struct { + Messages []Msg + DiscardMessages bool + + StartErr error + RcptErr map[string]error + BodyErr error + PartialBodyErr map[string]error + AbortErr error + CommitErr error + + InstName string +} + +/* +module.Module is implemented with dummy functions for logging done by MsgPipeline code. +*/ + +func (dt Target) Init(*config.Map) error { + return nil +} + +func (dt Target) InstanceName() string { + if dt.InstName != "" { + return dt.InstName + } + return "test_instance" +} + +func (dt Target) Name() string { + return "test_target" +} + +type testTargetDelivery struct { + msg Msg + tgt *Target +} + +type testTargetDeliveryPartial struct { + testTargetDelivery +} + +func (dt *Target) Start(ctx context.Context, msgMeta *module.MsgMetadata, mailFrom string) (module.Delivery, error) { + if dt.PartialBodyErr != nil { + return &testTargetDeliveryPartial{ + testTargetDelivery: testTargetDelivery{ + tgt: dt, + msg: Msg{MsgMeta: msgMeta, MailFrom: mailFrom}, + }, + }, dt.StartErr + } + return &testTargetDelivery{ + tgt: dt, + msg: Msg{MsgMeta: msgMeta, MailFrom: mailFrom}, + }, dt.StartErr +} + +func (dtd *testTargetDelivery) AddRcpt(ctx context.Context, to string, _ smtp.RcptOptions) error { + if dtd.tgt.RcptErr != nil { + if err := dtd.tgt.RcptErr[to]; err != nil { + return err + } + } + + dtd.msg.RcptTo = append(dtd.msg.RcptTo, to) + return nil +} + +func (dtd *testTargetDeliveryPartial) BodyNonAtomic(ctx context.Context, c module.StatusCollector, header textproto.Header, buf buffer.Buffer) { + if dtd.tgt.PartialBodyErr != nil { + for rcpt, err := range dtd.tgt.PartialBodyErr { + c.SetStatus(rcpt, err) + } + return + } + + dtd.msg.Header = header + + body, err := buf.Open() + if err != nil { + for rcpt, err := range dtd.tgt.PartialBodyErr { + c.SetStatus(rcpt, err) + } + return + } + defer body.Close() + + dtd.msg.Body, err = io.ReadAll(body) + if err != nil { + for rcpt, err := range dtd.tgt.PartialBodyErr { + c.SetStatus(rcpt, err) + } + } +} + +func (dtd *testTargetDelivery) Body(ctx context.Context, header textproto.Header, buf buffer.Buffer) error { + if dtd.tgt.PartialBodyErr != nil { + return errors.New("partial failure occurred, no additional information available") + } + if dtd.tgt.BodyErr != nil { + return dtd.tgt.BodyErr + } + + dtd.msg.Header = header + + body, err := buf.Open() + if err != nil { + return err + } + defer body.Close() + + if dtd.tgt.DiscardMessages { + // Don't bother. + _, err = io.Copy(io.Discard, body) + return err + } + + dtd.msg.Body, err = io.ReadAll(body) + return err +} + +func (dtd *testTargetDelivery) Abort(ctx context.Context) error { + return dtd.tgt.AbortErr +} + +func (dtd *testTargetDelivery) Commit(ctx context.Context) error { + if dtd.tgt.CommitErr != nil { + return dtd.tgt.CommitErr + } + if dtd.tgt.DiscardMessages { + return nil + } + dtd.tgt.Messages = append(dtd.tgt.Messages, dtd.msg) + return nil +} + +func DoTestDelivery(t *testing.T, tgt module.DeliveryTarget, from string, to []string) string { + t.Helper() + return DoTestDeliveryMeta(t, tgt, from, to, &module.MsgMetadata{ + OriginalFrom: from, + }) +} + +func DoTestDeliveryMeta(t *testing.T, tgt module.DeliveryTarget, from string, to []string, msgMeta *module.MsgMetadata) string { + t.Helper() + + id, err := DoTestDeliveryErrMeta(t, tgt, from, to, msgMeta) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + return id +} + +func DoTestDeliveryNonAtomic(t *testing.T, c module.StatusCollector, tgt module.DeliveryTarget, from string, to []string) string { + t.Helper() + + IDRaw := sha1.Sum([]byte(t.Name())) + encodedID := hex.EncodeToString(IDRaw[:]) + + testCtx := context.Background() + + body := buffer.MemoryBuffer{Slice: []byte("foobar\r\n")} + msgMeta := module.MsgMetadata{ + DontTraceSender: true, + ID: encodedID, + OriginalFrom: from, + } + t.Log("-- tgt.Start", from) + delivery, err := tgt.Start(testCtx, &msgMeta, from) + if err != nil { + t.Log("-- ... tgt.Start", from, err, exterrors.Fields(err)) + t.Fatalf("Unexpected err: %v %+v", err, exterrors.Fields(err)) + return encodedID + } + for _, rcpt := range to { + t.Log("-- delivery.AddRcpt", rcpt) + if err := delivery.AddRcpt(testCtx, rcpt, smtp.RcptOptions{}); err != nil { + t.Log("-- ... delivery.AddRcpt", rcpt, err, exterrors.Fields(err)) + t.Log("-- delivery.Abort") + if err := delivery.Abort(testCtx); err != nil { + t.Log("-- delivery.Abort:", err, exterrors.Fields(err)) + } + t.Fatalf("Unexpected err: %v %+v", err, exterrors.Fields(err)) + return encodedID + } + } + t.Log("-- delivery.BodyNonAtomic") + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + delivery.(module.PartialDelivery).BodyNonAtomic(testCtx, c, hdr, body) + t.Log("-- delivery.Commit") + if err := delivery.Commit(testCtx); err != nil { + t.Fatalf("Unexpected err: %v %+v", err, exterrors.Fields(err)) + } + + return encodedID +} + +const DeliveryData = "A: 1\r\n" + + "B: 2\r\n" + + "\r\n" + + "foobar\r\n" + +func DoTestDeliveryErr(t *testing.T, tgt module.DeliveryTarget, from string, to []string) (string, error) { + return DoTestDeliveryErrMeta(t, tgt, from, to, &module.MsgMetadata{}) +} + +func DoTestDeliveryErrMeta(t *testing.T, tgt module.DeliveryTarget, from string, to []string, msgMeta *module.MsgMetadata) (string, error) { + t.Helper() + + IDRaw := sha1.Sum([]byte(t.Name())) + encodedID := hex.EncodeToString(IDRaw[:]) + testCtx := context.Background() + + body := buffer.MemoryBuffer{Slice: []byte("foobar\r\n")} + msgMeta.DontTraceSender = true + msgMeta.ID = encodedID + t.Log("-- tgt.Start", from) + delivery, err := tgt.Start(testCtx, msgMeta, from) + if err != nil { + t.Log("-- ... tgt.Start", from, err, exterrors.Fields(err)) + return encodedID, err + } + for _, rcpt := range to { + t.Log("-- delivery.AddRcpt", rcpt) + if err := delivery.AddRcpt(testCtx, rcpt, smtp.RcptOptions{}); err != nil { + t.Log("-- ... delivery.AddRcpt", rcpt, err, exterrors.Fields(err)) + t.Log("-- delivery.Abort") + if err := delivery.Abort(testCtx); err != nil { + t.Log("-- delivery.Abort:", err, exterrors.Fields(err)) + } + return encodedID, err + } + } + t.Log("-- delivery.Body") + hdr := textproto.Header{} + hdr.Add("B", "2") + hdr.Add("A", "1") + if err := delivery.Body(testCtx, hdr, body); err != nil { + t.Log("-- ... delivery.Body", err, exterrors.Fields(err)) + t.Log("-- delivery.Abort") + if err := delivery.Abort(testCtx); err != nil { + t.Log("-- ... delivery.Abort:", err, exterrors.Fields(err)) + } + return encodedID, err + } + t.Log("-- delivery.Commit") + if err := delivery.Commit(testCtx); err != nil { + t.Log("-- ... delivery.Commit", err, exterrors.Fields(err)) + return encodedID, err + } + + return encodedID, err +} + +func CheckTestMessage(t *testing.T, tgt *Target, indx int, sender string, rcpt []string) { + t.Helper() + + if len(tgt.Messages) <= indx { + t.Errorf("wrong amount of messages received, want at least %d, got %d", indx+1, len(tgt.Messages)) + return + } + msg := tgt.Messages[indx] + + CheckMsg(t, &msg, sender, rcpt) +} + +func CheckMsg(t *testing.T, msg *Msg, sender string, rcpt []string) { + t.Helper() + + idRaw := sha1.Sum([]byte(t.Name())) + encodedId := hex.EncodeToString(idRaw[:]) + + CheckMsgID(t, msg, sender, rcpt, encodedId) +} + +func CheckMsgID(t *testing.T, msg *Msg, sender string, rcpt []string, id string) string { + t.Helper() + + if msg.MsgMeta.ID != id && id != "" { + t.Errorf("empty or wrong delivery context for passed message? %+v", msg.MsgMeta) + } + if msg.MailFrom != sender { + t.Errorf("wrong sender, want %s, got %s", sender, msg.MailFrom) + } + + sort.Strings(rcpt) + sort.Strings(msg.RcptTo) + if !reflect.DeepEqual(msg.RcptTo, rcpt) { + t.Errorf("wrong recipients, want %v, got %v", rcpt, msg.RcptTo) + } + if string(msg.Body) != "foobar\r\n" { + t.Errorf("wrong body, want '%s', got '%s' (%v)", "foobar\r\n", string(msg.Body), msg.Body) + } + + return msg.MsgMeta.ID +} diff --git a/internal/tls/acme/acme.go b/internal/tls/acme/acme.go new file mode 100644 index 0000000..a09c3e0 --- /dev/null +++ b/internal/tls/acme/acme.go @@ -0,0 +1,164 @@ +package acme + +import ( + "context" + "crypto/tls" + "fmt" + "path/filepath" + + "github.com/caddyserver/certmagic" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +const modName = "tls.loader.acme" + +type Loader struct { + instName string + + store certmagic.Storage + cache *certmagic.Cache + cfg *certmagic.Config + cancelManage context.CancelFunc + + log log.Logger +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, fmt.Errorf("%s: no inline args expected", modName) + } + return &Loader{ + instName: instName, + log: log.Logger{Name: modName}, + }, nil +} + +func (l *Loader) Init(cfg *config.Map) error { + var ( + hostname string + extraNames []string + storePath string + caPath string + testCAPath string + email string + agreed bool + challenge string + overrideDomain string + provider certmagic.DNSProvider + ) + cfg.Bool("debug", true, false, &l.log.Debug) + cfg.String("hostname", true, true, "", &hostname) + cfg.StringList("extra_names", false, false, nil, &extraNames) + cfg.String("store_path", false, false, + filepath.Join(config.StateDirectory, "acme"), &storePath) + cfg.String("ca", false, false, + certmagic.LetsEncryptProductionCA, &caPath) + cfg.String("test_ca", false, false, + certmagic.LetsEncryptStagingCA, &testCAPath) + cfg.String("email", false, false, + "", &email) + cfg.String("override_domain", false, false, + "", &overrideDomain) + cfg.Bool("agreed", false, false, &agreed) + cfg.Enum("challenge", false, true, + []string{"dns-01"}, "dns-01", &challenge) + cfg.Custom("dns", false, false, func() (interface{}, error) { + return nil, nil + }, func(m *config.Map, node config.Node) (interface{}, error) { + var p certmagic.DNSProvider + err := modconfig.ModuleFromNode("libdns", node.Args, node, m.Globals, &p) + return p, err + }, &provider) + if _, err := cfg.Process(); err != nil { + return err + } + + cmLog := l.log.Zap() + + l.store = &certmagic.FileStorage{Path: storePath} + l.cache = certmagic.NewCache(certmagic.CacheOptions{ + Logger: cmLog, + GetConfigForCert: func(c certmagic.Certificate) (*certmagic.Config, error) { + return l.cfg, nil + }, + }) + + l.cfg = certmagic.New(l.cache, certmagic.Config{ + Storage: l.store, // not sure if it is necessary to set these twice + Logger: cmLog, + DefaultServerName: hostname, + }) + issuer := certmagic.NewACMEIssuer(l.cfg, certmagic.ACMEIssuer{ + Logger: cmLog, + CA: caPath, + TestCA: testCAPath, + Email: email, + Agreed: agreed, + }) + + switch challenge { + case "dns-01": + issuer.DisableTLSALPNChallenge = true + issuer.DisableHTTPChallenge = true + if provider == nil { + return fmt.Errorf("tls.loader.acme: dns-01 challenge requires a configured DNS provider") + } + issuer.DNS01Solver = &certmagic.DNS01Solver{ + DNSManager: certmagic.DNSManager{ + DNSProvider: provider, + OverrideDomain: overrideDomain, + }, + } + default: + return fmt.Errorf("tls.loader.acme: challenge not supported") + } + l.cfg.Issuers = []certmagic.Issuer{issuer} + + if module.NoRun { + return nil + } + + manageCtx, cancelManage := context.WithCancel(context.Background()) + err := l.cfg.ManageAsync(manageCtx, append([]string{hostname}, extraNames...)) + if err != nil { + cancelManage() + return err + } + l.cancelManage = cancelManage + + return nil +} + +func (l *Loader) ConfigureTLS(c *tls.Config) error { + c.GetCertificate = l.cfg.GetCertificate + return nil +} + +func (l *Loader) Close() error { + l.cancelManage() + l.cache.Stop() + return nil +} + +func (l *Loader) Name() string { + return modName +} + +func (l *Loader) InstanceName() string { + return l.instName +} + +func init() { + hooks.AddHook(hooks.EventShutdown, func() { + certmagic.CleanUpOwnLocks(context.TODO(), log.DefaultLogger.Zap()) + }) +} + +func init() { + var _ module.TLSLoader = &Loader{} + module.Register(modName, New) +} diff --git a/internal/tls/file.go b/internal/tls/file.go new file mode 100644 index 0000000..943a59e --- /dev/null +++ b/internal/tls/file.go @@ -0,0 +1,168 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tls + +import ( + "crypto/tls" + "errors" + "fmt" + "path/filepath" + "sync" + "time" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" +) + +type FileLoader struct { + instName string + inlineArgs []string + certPaths []string + keyPaths []string + log log.Logger + + certs []tls.Certificate + certsLock sync.RWMutex + + reloadTick *time.Ticker + stopTick chan struct{} +} + +func NewFileLoader(_, instName string, _, inlineArgs []string) (module.Module, error) { + return &FileLoader{ + instName: instName, + inlineArgs: inlineArgs, + log: log.Logger{Name: "tls.loader.file", Debug: log.DefaultLogger.Debug}, + stopTick: make(chan struct{}), + }, nil +} + +func (f *FileLoader) Init(cfg *config.Map) error { + cfg.StringList("certs", false, false, nil, &f.certPaths) + cfg.StringList("keys", false, false, nil, &f.keyPaths) + if _, err := cfg.Process(); err != nil { + return err + } + + if len(f.certPaths) != len(f.keyPaths) { + return errors.New("tls.loader.file: mismatch in certs and keys count") + } + + if len(f.inlineArgs)%2 != 0 { + return errors.New("tls.loader.file: odd amount of arguments") + } + for i := 0; i < len(f.inlineArgs); i += 2 { + f.certPaths = append(f.certPaths, f.inlineArgs[i]) + f.keyPaths = append(f.keyPaths, f.inlineArgs[i+1]) + } + + for _, certPath := range f.certPaths { + if !filepath.IsAbs(certPath) { + return fmt.Errorf("tls.loader.file: only absolute paths allowed in certificate paths: sorry :(") + } + } + + if err := f.loadCerts(); err != nil { + return err + } + + hooks.AddHook(hooks.EventReload, func() { + f.log.Println("reloading certificates") + if err := f.loadCerts(); err != nil { + f.log.Error("reload failed", err) + } + }) + + f.reloadTick = time.NewTicker(time.Minute) + go f.reloadTicker() + return nil +} + +func (f *FileLoader) Close() error { + f.reloadTick.Stop() + f.stopTick <- struct{}{} + return nil +} + +func (f *FileLoader) Name() string { + return "tls.loader.file" +} + +func (f *FileLoader) InstanceName() string { + return f.instName +} + +func (f *FileLoader) reloadTicker() { + for { + select { + case <-f.reloadTick.C: + f.log.Debugln("reloading certs") + if err := f.loadCerts(); err != nil { + f.log.Error("reload failed", err) + } + case <-f.stopTick: + return + } + } +} + +func (f *FileLoader) loadCerts() error { + if len(f.certPaths) != len(f.keyPaths) { + return errors.New("mismatch in certs and keys count") + } + + if len(f.certPaths) == 0 { + return errors.New("tls.loader.file: at least one certificate required") + } + + certs := make([]tls.Certificate, 0, len(f.certPaths)) + + for i := range f.certPaths { + certPath := f.certPaths[i] + keyPath := f.keyPaths[i] + + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return fmt.Errorf("failed to load %s and %s: %v", certPath, keyPath, err) + } + certs = append(certs, cert) + } + + f.certsLock.Lock() + defer f.certsLock.Unlock() + f.certs = certs + + return nil +} + +func (f *FileLoader) ConfigureTLS(c *tls.Config) error { + // Loader function replaces only the whole slice. + f.certsLock.RLock() + defer f.certsLock.RUnlock() + + c.Certificates = f.certs + return nil +} + +func init() { + var _ module.TLSLoader = &FileLoader{} + module.Register("tls.loader.file", NewFileLoader) +} diff --git a/internal/tls/self_signed.go b/internal/tls/self_signed.go new file mode 100644 index 0000000..d5e174f --- /dev/null +++ b/internal/tls/self_signed.go @@ -0,0 +1,112 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tls + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net" + "time" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +type SelfSignedLoader struct { + instName string + serverNames []string + + cert tls.Certificate +} + +func NewSelfSignedLoader(_, instName string, _, inlineArgs []string) (module.Module, error) { + return &SelfSignedLoader{ + instName: instName, + serverNames: inlineArgs, + }, nil +} + +func (f *SelfSignedLoader) Init(cfg *config.Map) error { + if _, err := cfg.Process(); err != nil { + return err + } + + privKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return err + } + + notBefore := time.Now() + notAfter := notBefore.Add(24 * time.Hour * 7) + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return err + } + cert := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{Organization: []string{"Maddy Self-Signed"}}, + NotBefore: notBefore, + NotAfter: notAfter, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + + for _, name := range f.serverNames { + if ip := net.ParseIP(name); ip != nil { + cert.IPAddresses = append(cert.IPAddresses, ip) + } else { + cert.DNSNames = append(cert.DNSNames, name) + } + } + derBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &privKey.PublicKey, privKey) + if err != nil { + return err + } + + f.cert = tls.Certificate{ + Certificate: [][]byte{derBytes}, + PrivateKey: privKey, + Leaf: cert, + } + return nil +} + +func (f *SelfSignedLoader) Name() string { + return "tls.loader.self_signed" +} + +func (f *SelfSignedLoader) InstanceName() string { + return f.instName +} + +func (f *SelfSignedLoader) ConfigureTLS(c *tls.Config) error { + c.Certificates = []tls.Certificate{f.cert} + return nil +} + +func init() { + var _ module.TLSLoader = &SelfSignedLoader{} + module.Register("tls.loader.self_signed", NewSelfSignedLoader) +} diff --git a/internal/updatepipe/backend.go b/internal/updatepipe/backend.go new file mode 100644 index 0000000..ba04df7 --- /dev/null +++ b/internal/updatepipe/backend.go @@ -0,0 +1,45 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package updatepipe + +type BackendMode int + +const ( + // ModeReplicate configures backend to both send and receive updates over + // the pipe. + ModeReplicate BackendMode = iota + + // ModePush configures backend to send updates over the pipe only. + // + // If EnableUpdatePipe(ModePush) is called for backend, its Updates() + // channel will never receive any updates. + ModePush BackendMode = iota +) + +// The Backend interface is implemented by storage backends that support both +// updates serialization using the internal updatepipe.P implementation. +// To activate this implementation, EnableUpdatePipe should be called. +type Backend interface { + // EnableUpdatePipe enables the internal update pipe implementation. + // The mode argument selects the pipe behavior. EnableUpdatePipe must be + // called before the first call to the Updates() method. + // + // This method is idempotent. All calls after a successful one do nothing. + EnableUpdatePipe(mode BackendMode) error +} diff --git a/internal/updatepipe/pubsub/pq.go b/internal/updatepipe/pubsub/pq.go new file mode 100644 index 0000000..29f9c52 --- /dev/null +++ b/internal/updatepipe/pubsub/pq.go @@ -0,0 +1,86 @@ +package pubsub + +import ( + "context" + "database/sql" + "time" + + "github.com/foxcpp/maddy/framework/log" + "github.com/lib/pq" +) + +type Msg struct { + Key string + Payload string +} + +type PqPubSub struct { + Notify chan Msg + + L *pq.Listener + sender *sql.DB + + Log log.Logger +} + +func NewPQ(dsn string) (*PqPubSub, error) { + l := &PqPubSub{ + Log: log.Logger{Name: "pgpubsub"}, + Notify: make(chan Msg), + } + l.L = pq.NewListener(dsn, 10*time.Second, time.Minute, l.eventHandler) + var err error + l.sender, err = sql.Open("postgres", dsn) + if err != nil { + return nil, err + } + + go func() { + defer close(l.Notify) + for n := range l.L.Notify { + if n == nil { + continue + } + + l.Notify <- Msg{Key: n.Channel, Payload: n.Extra} + } + }() + + return l, nil +} + +func (l *PqPubSub) Close() error { + l.sender.Close() + l.L.Close() + return nil +} + +func (l *PqPubSub) eventHandler(ev pq.ListenerEventType, err error) { + switch ev { + case pq.ListenerEventConnected: + l.Log.DebugMsg("connected") + case pq.ListenerEventReconnected: + l.Log.Msg("connection reestablished") + case pq.ListenerEventConnectionAttemptFailed: + l.Log.Error("connection attempt failed", err) + case pq.ListenerEventDisconnected: + l.Log.Msg("connection closed", "err", err) + } +} + +func (l *PqPubSub) Subscribe(_ context.Context, key string) error { + return l.L.Listen(key) +} + +func (l *PqPubSub) Unsubscribe(_ context.Context, key string) error { + return l.L.Unlisten(key) +} + +func (l *PqPubSub) Publish(key, payload string) error { + _, err := l.sender.Exec(`SELECT pg_notify($1, $2)`, key, payload) + return err +} + +func (l *PqPubSub) Listener() chan Msg { + return l.Notify +} diff --git a/internal/updatepipe/pubsub/pubsub.go b/internal/updatepipe/pubsub/pubsub.go new file mode 100644 index 0000000..64480ab --- /dev/null +++ b/internal/updatepipe/pubsub/pubsub.go @@ -0,0 +1,11 @@ +package pubsub + +import "context" + +type PubSub interface { + Subscribe(ctx context.Context, key string) error + Unsubscribe(ctx context.Context, key string) error + Publish(key, payload string) error + Listener() chan Msg + Close() error +} diff --git a/internal/updatepipe/pubsub_pipe.go b/internal/updatepipe/pubsub_pipe.go new file mode 100644 index 0000000..b1cef67 --- /dev/null +++ b/internal/updatepipe/pubsub_pipe.go @@ -0,0 +1,101 @@ +package updatepipe + +import ( + "context" + "fmt" + "os" + "strconv" + + mess "github.com/foxcpp/go-imap-mess" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/internal/updatepipe/pubsub" +) + +type PubSubPipe struct { + PubSub pubsub.PubSub + Log log.Logger +} + +func (p *PubSubPipe) Listen(upds chan<- mess.Update) error { + go func() { + for m := range p.PubSub.Listener() { + id, upd, err := parseUpdate(m.Payload) + if err != nil { + p.Log.Error("failed to parse update", err) + continue + } + if id == p.myID() { + continue + } + upds <- *upd + } + }() + return nil +} + +func (p *PubSubPipe) InitPush() error { + return nil +} + +func (p *PubSubPipe) myID() string { + return fmt.Sprintf("%d-%p", os.Getpid(), p) +} + +func (p *PubSubPipe) channel(key interface{}) (string, error) { + var psKey string + switch k := key.(type) { + case string: + psKey = k + case uint64: + psKey = "__uint64_" + strconv.FormatUint(k, 10) + default: + return "", fmt.Errorf("updatepipe: key type must be either string or uint64") + } + return psKey, nil +} + +func (p *PubSubPipe) Subscribe(key interface{}) { + psKey, err := p.channel(key) + if err != nil { + p.Log.Error("invalid key passed to Subscribe", err) + return + } + + if err := p.PubSub.Subscribe(context.TODO(), psKey); err != nil { + p.Log.Error("pubsub subscribe failed", err) + } else { + p.Log.DebugMsg("subscribed to pubsub", "channel", psKey) + } +} + +func (p *PubSubPipe) Unsubscribe(key interface{}) { + psKey, err := p.channel(key) + if err != nil { + p.Log.Error("invalid key passed to Unsubscribe", err) + return + } + + if err := p.PubSub.Unsubscribe(context.TODO(), psKey); err != nil { + p.Log.Error("pubsub unsubscribe failed", err) + } else { + p.Log.DebugMsg("unsubscribed from pubsub", "channel", psKey) + } +} + +func (p *PubSubPipe) Push(upd mess.Update) error { + psKey, err := p.channel(upd.Key) + if err != nil { + return err + } + + updBlob, err := formatUpdate(p.myID(), upd) + if err != nil { + return err + } + + return p.PubSub.Publish(psKey, updBlob) +} + +func (p *PubSubPipe) Close() error { + return p.PubSub.Close() +} diff --git a/internal/updatepipe/serialize.go b/internal/updatepipe/serialize.go new file mode 100644 index 0000000..d5a941f --- /dev/null +++ b/internal/updatepipe/serialize.go @@ -0,0 +1,69 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package updatepipe + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + mess "github.com/foxcpp/go-imap-mess" +) + +func unescapeName(s string) string { + return strings.ReplaceAll(s, "\x10", ";") +} + +func escapeName(s string) string { + return strings.ReplaceAll(s, ";", "\x10") +} + +func parseUpdate(s string) (id string, upd *mess.Update, err error) { + parts := strings.SplitN(s, ";", 2) + if len(parts) != 2 { + return "", nil, errors.New("updatepipe: mismatched parts count") + } + + upd = &mess.Update{} + dec := json.NewDecoder(strings.NewReader(unescapeName(parts[1]))) + dec.UseNumber() + err = dec.Decode(upd) + if err != nil { + return "", nil, fmt.Errorf("parseUpdate: %w", err) + } + + if val, ok := upd.Key.(json.Number); ok { + upd.Key, _ = strconv.ParseUint(val.String(), 10, 64) + } + + return parts[0], upd, nil +} + +func formatUpdate(myID string, upd mess.Update) (string, error) { + updBlob, err := json.Marshal(upd) + if err != nil { + return "", fmt.Errorf("formatUpdate: %w", err) + } + return strings.Join([]string{ + myID, + escapeName(string(updBlob)), + }, ";") + "\n", nil +} diff --git a/internal/updatepipe/unix_pipe.go b/internal/updatepipe/unix_pipe.go new file mode 100644 index 0000000..a8249f9 --- /dev/null +++ b/internal/updatepipe/unix_pipe.go @@ -0,0 +1,129 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package updatepipe + +import ( + "bufio" + "fmt" + "io" + "net" + "os" + + mess "github.com/foxcpp/go-imap-mess" + "github.com/foxcpp/maddy/framework/log" +) + +// UnixSockPipe implements the UpdatePipe interface by serializating updates +// to/from a Unix domain socket. Due to the way Unix sockets work, only one +// Listen goroutine can be running. +// +// The socket is stream-oriented and consists of the following messages: +// +// SENDER_ID;JSON_SERIALIZED_INTERNAL_OBJECT\n +// +// And SENDER_ID is Process ID and UnixSockPipe address concated as a string. +// It is used to deduplicate updates sent to Push and recevied via Listen. +// +// The SockPath field specifies the socket path to use. The actual socket +// is initialized on the first call to Listen or (Init)Push. +type UnixSockPipe struct { + SockPath string + Log log.Logger + + listener net.Listener + sender net.Conn +} + +var _ P = &UnixSockPipe{} + +func (usp *UnixSockPipe) myID() string { + return fmt.Sprintf("%d-%p", os.Getpid(), usp) +} + +func (usp *UnixSockPipe) readUpdates(conn net.Conn, updCh chan<- mess.Update) { + scnr := bufio.NewScanner(conn) + for scnr.Scan() { + id, upd, err := parseUpdate(scnr.Text()) + if err != nil { + usp.Log.Error("malformed update received", err, "str", scnr.Text()) + } + + // It is our own update, skip. + if id == usp.myID() { + continue + } + + updCh <- *upd + } +} + +func (usp *UnixSockPipe) Listen(upd chan<- mess.Update) error { + l, err := net.Listen("unix", usp.SockPath) + if err != nil { + return err + } + usp.listener = l + go func() { + for { + conn, err := l.Accept() + if err != nil { + return + } + go usp.readUpdates(conn, upd) + } + }() + return nil +} + +func (usp *UnixSockPipe) InitPush() error { + sock, err := net.Dial("unix", usp.SockPath) + if err != nil { + return err + } + + usp.sender = sock + return nil +} + +func (usp *UnixSockPipe) Push(upd mess.Update) error { + if usp.sender == nil { + if err := usp.InitPush(); err != nil { + return err + } + } + + updStr, err := formatUpdate(usp.myID(), upd) + if err != nil { + return err + } + + _, err = io.WriteString(usp.sender, updStr) + return err +} + +func (usp *UnixSockPipe) Close() error { + if usp.sender != nil { + usp.sender.Close() + } + if usp.listener != nil { + usp.listener.Close() + os.Remove(usp.SockPath) + } + return nil +} diff --git a/internal/updatepipe/update_pipe.go b/internal/updatepipe/update_pipe.go new file mode 100644 index 0000000..1427c22 --- /dev/null +++ b/internal/updatepipe/update_pipe.go @@ -0,0 +1,62 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package updatepipe implements utilities for serialization and transport of +// IMAP update objects between processes and machines. +// +// Its main goal is provide maddy command with ability to properly notify the +// server about changes without relying on it to coordinate access in the +// first place (so maddy command can work without a running server or with a +// broken server instance). +// +// Additionally, it can be used to transfer IMAP updates between replicated +// nodes. +package updatepipe + +import ( + mess "github.com/foxcpp/go-imap-mess" +) + +// The P interface represents the handle for a transport medium used for IMAP +// updates. +type P interface { + // Listen starts the "pull" goroutine that reads updates from the pipe and + // sends them to the channel. + // + // Usually it is not possible to call Listen multiple times for the same + // pipe. + // + // Updates sent using the same UpdatePipe object using Push are not + // duplicates to the channel passed to Listen. + Listen(upds chan<- mess.Update) error + + // InitPush prepares the UpdatePipe to be used as updates source (Push + // method). + // + // It is called implicitly on the first Push call, but calling it + // explicitly allows to detect initialization errors early. + InitPush() error + + // Push writes the update to the pipe. + // + // The update will not be duplicated if the UpdatePipe is also listening + // for updates. + Push(upd mess.Update) error + + Close() error +} diff --git a/maddy.conf b/maddy.conf new file mode 100644 index 0000000..5f02fb3 --- /dev/null +++ b/maddy.conf @@ -0,0 +1,181 @@ +## Maddy Mail Server - default configuration file (2022-06-18) +# Suitable for small-scale deployments. Uses its own format for local users DB, +# should be managed via maddy subcommands. +# +# See tutorials at https://maddy.email for guidance on typical +# configuration changes. + +# ---------------------------------------------------------------------------- +# Base variables + +$(hostname) = example.org +$(primary_domain) = example.org +$(local_domains) = $(primary_domain) + +tls file /etc/maddy/certs/$(hostname)/fullchain.pem /etc/maddy/certs/$(hostname)/privkey.pem + +# ---------------------------------------------------------------------------- +# Local storage & authentication + +# pass_table provides local hashed passwords storage for authentication of +# users. It can be configured to use any "table" module, in default +# configuration a table in SQLite DB is used. +# Table can be replaced to use e.g. a file for passwords. Or pass_table module +# can be replaced altogether to use some external source of credentials (e.g. +# PAM, /etc/shadow file). +# +# If table module supports it (sql_table does) - credentials can be managed +# using 'maddy creds' command. + +auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } +} + +# imapsql module stores all indexes and metadata necessary for IMAP using a +# relational database. It is used by IMAP endpoint for mailbox access and +# also by SMTP & Submission endpoints for delivery of local messages. +# +# IMAP accounts, mailboxes and all message metadata can be inspected using +# imap-* subcommands of maddy. + +storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db +} + +# ---------------------------------------------------------------------------- +# SMTP endpoints + message routing + +hostname $(hostname) + +table.chain local_rewrites { + optional_step regexp "(.+)\+(.+)@(.+)" "$1@$3" + optional_step static { + entry postmaster postmaster@$(primary_domain) + } + optional_step file /etc/maddy/aliases +} + +msgpipeline local_routing { + # Insert handling for special-purpose local domains here. + # e.g. + # destination lists.example.org { + # deliver_to lmtp tcp://127.0.0.1:8024 + # } + + destination postmaster $(local_domains) { + modify { + replace_rcpt &local_rewrites + } + + deliver_to &local_mailboxes + } + + default_destination { + reject 550 5.1.1 "User doesn't exist" + } +} + +smtp tcp://0.0.0.0:25 { + limits { + # Up to 20 msgs/sec across max. 10 SMTP connections. + all rate 20 1s + all concurrency 10 + } + + dmarc yes + check { + require_mx_record + dkim + spf + } + + source $(local_domains) { + reject 501 5.1.8 "Use Submission for outgoing SMTP" + } + default_source { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + reject 550 5.1.1 "User doesn't exist" + } + } +} + +submission tls://0.0.0.0:465 tcp://0.0.0.0:587 { + limits { + # Up to 50 msgs/sec across any amount of SMTP connections. + all rate 50 1s + } + + auth &local_authdb + + source $(local_domains) { + check { + authorize_sender { + prepare_email &local_rewrites + user_to_email identity + } + } + + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + modify { + dkim $(primary_domain) $(local_domains) default + } + deliver_to &remote_queue + } + } + default_source { + reject 501 5.1.8 "Non-local sender domain" + } +} + +target.remote outbound_delivery { + limits { + # Up to 20 msgs/sec across max. 10 SMTP connections + # for each recipient domain. + destination rate 20 1s + destination concurrency 10 + } + mx_auth { + dane + mtasts { + cache fs + fs_dir mtasts_cache/ + } + local_policy { + min_tls_level encrypted + min_mx_level none + } + } +} + +target.queue remote_queue { + target &outbound_delivery + + autogenerated_msg_domain $(primary_domain) + bounce { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + reject 550 5.0.0 "Refusing to send DSNs to non-local addresses" + } + } +} + +# ---------------------------------------------------------------------------- +# IMAP endpoints + +imap tls://0.0.0.0:993 tcp://0.0.0.0:143 { + auth &local_authdb + storage &local_mailboxes +} diff --git a/maddy.conf.docker b/maddy.conf.docker new file mode 100644 index 0000000..39c3bbb --- /dev/null +++ b/maddy.conf.docker @@ -0,0 +1,182 @@ +## Maddy Mail Server - default configuration file (2022-06-18) +## This is the copy of maddy.conf with changes necessary to run it in Docker. +# Suitable for small-scale deployments. Uses its own format for local users DB, +# should be managed via maddy subcommands. +# +# See tutorials at https://maddy.email for guidance on typical +# configuration changes. + +# ---------------------------------------------------------------------------- +# Base variables + +$(hostname) = {env:MADDY_HOSTNAME} +$(primary_domain) = {env:MADDY_DOMAIN} +$(local_domains) = $(primary_domain) + +tls file /data/tls/fullchain.pem /data/tls/privkey.pem + +# ---------------------------------------------------------------------------- +# Local storage & authentication + +# pass_table provides local hashed passwords storage for authentication of +# users. It can be configured to use any "table" module, in default +# configuration a table in SQLite DB is used. +# Table can be replaced to use e.g. a file for passwords. Or pass_table module +# can be replaced altogether to use some external source of credentials (e.g. +# PAM, /etc/shadow file). +# +# If table module supports it (sql_table does) - credentials can be managed +# using 'maddy creds' command. + +auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } +} + +# imapsql module stores all indexes and metadata necessary for IMAP using a +# relational database. It is used by IMAP endpoint for mailbox access and +# also by SMTP & Submission endpoints for delivery of local messages. +# +# IMAP accounts, mailboxes and all message metadata can be inspected using +# imap-* subcommands of maddy. + +storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db +} + +# ---------------------------------------------------------------------------- +# SMTP endpoints + message routing + +hostname $(hostname) + +table.chain local_rewrites { + optional_step regexp "(.+)\+(.+)@(.+)" "$1@$3" + optional_step static { + entry postmaster postmaster@$(primary_domain) + } + optional_step file /etc/maddy/aliases +} + +msgpipeline local_routing { + # Insert handling for special-purpose local domains here. + # e.g. + # destination lists.example.org { + # deliver_to lmtp tcp://127.0.0.1:8024 + # } + + destination postmaster $(local_domains) { + modify { + replace_rcpt &local_rewrites + } + + deliver_to &local_mailboxes + } + + default_destination { + reject 550 5.1.1 "User doesn't exist" + } +} + +smtp tcp://0.0.0.0:25 { + limits { + # Up to 20 msgs/sec across max. 10 SMTP connections. + all rate 20 1s + all concurrency 10 + } + + dmarc yes + check { + require_mx_record + dkim + spf + } + + source $(local_domains) { + reject 501 5.1.8 "Use Submission for outgoing SMTP" + } + default_source { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + reject 550 5.1.1 "User doesn't exist" + } + } +} + +submission tls://0.0.0.0:465 tcp://0.0.0.0:587 { + limits { + # Up to 50 msgs/sec across any amount of SMTP connections. + all rate 50 1s + } + + auth &local_authdb + + source $(local_domains) { + check { + authorize_sender { + prepare_email &local_rewrites + user_to_email identity + } + } + + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + modify { + dkim $(primary_domain) $(local_domains) default + } + deliver_to &remote_queue + } + } + default_source { + reject 501 5.1.8 "Non-local sender domain" + } +} + +target.remote outbound_delivery { + limits { + # Up to 20 msgs/sec across max. 10 SMTP connections + # for each recipient domain. + destination rate 20 1s + destination concurrency 10 + } + mx_auth { + dane + mtasts { + cache fs + fs_dir mtasts_cache/ + } + local_policy { + min_tls_level encrypted + min_mx_level none + } + } +} + +target.queue remote_queue { + target &outbound_delivery + + autogenerated_msg_domain $(primary_domain) + bounce { + destination postmaster $(local_domains) { + deliver_to &local_routing + } + default_destination { + reject 550 5.0.0 "Refusing to send DSNs to non-local addresses" + } + } +} + +# ---------------------------------------------------------------------------- +# IMAP endpoints + +imap tls://0.0.0.0:993 tcp://0.0.0.0:143 { + auth &local_authdb + storage &local_mailboxes +} diff --git a/maddy.go b/maddy.go new file mode 100644 index 0000000..5439fba --- /dev/null +++ b/maddy.go @@ -0,0 +1,434 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package maddy + +import ( + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "runtime" + "runtime/debug" + + "github.com/caddyserver/certmagic" + parser "github.com/foxcpp/maddy/framework/cfgparser" + "github.com/foxcpp/maddy/framework/config" + modconfig "github.com/foxcpp/maddy/framework/config/module" + "github.com/foxcpp/maddy/framework/config/tls" + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/authz" + maddycli "github.com/foxcpp/maddy/internal/cli" + "github.com/urfave/cli/v2" + + // Import packages for side-effect of module registration. + _ "github.com/foxcpp/maddy/internal/auth/dovecot_sasl" + _ "github.com/foxcpp/maddy/internal/auth/external" + _ "github.com/foxcpp/maddy/internal/auth/ldap" + _ "github.com/foxcpp/maddy/internal/auth/netauth" + _ "github.com/foxcpp/maddy/internal/auth/pam" + _ "github.com/foxcpp/maddy/internal/auth/pass_table" + _ "github.com/foxcpp/maddy/internal/auth/plain_separate" + _ "github.com/foxcpp/maddy/internal/auth/shadow" + _ "github.com/foxcpp/maddy/internal/check/authorize_sender" + _ "github.com/foxcpp/maddy/internal/check/command" + _ "github.com/foxcpp/maddy/internal/check/dkim" + _ "github.com/foxcpp/maddy/internal/check/dns" + _ "github.com/foxcpp/maddy/internal/check/dnsbl" + _ "github.com/foxcpp/maddy/internal/check/milter" + _ "github.com/foxcpp/maddy/internal/check/requiretls" + _ "github.com/foxcpp/maddy/internal/check/rspamd" + _ "github.com/foxcpp/maddy/internal/check/spf" + _ "github.com/foxcpp/maddy/internal/endpoint/dovecot_sasld" + _ "github.com/foxcpp/maddy/internal/endpoint/imap" + _ "github.com/foxcpp/maddy/internal/endpoint/openmetrics" + _ "github.com/foxcpp/maddy/internal/endpoint/smtp" + _ "github.com/foxcpp/maddy/internal/imap_filter" + _ "github.com/foxcpp/maddy/internal/imap_filter/command" + _ "github.com/foxcpp/maddy/internal/libdns" + _ "github.com/foxcpp/maddy/internal/modify" + _ "github.com/foxcpp/maddy/internal/modify/dkim" + _ "github.com/foxcpp/maddy/internal/storage/blob/fs" + _ "github.com/foxcpp/maddy/internal/storage/blob/s3" + _ "github.com/foxcpp/maddy/internal/storage/imapsql" + _ "github.com/foxcpp/maddy/internal/table" + _ "github.com/foxcpp/maddy/internal/target/queue" + _ "github.com/foxcpp/maddy/internal/target/remote" + _ "github.com/foxcpp/maddy/internal/target/smtp" + _ "github.com/foxcpp/maddy/internal/tls" + _ "github.com/foxcpp/maddy/internal/tls/acme" +) + +var ( + Version = "go-build" + + enableDebugFlags = false +) + +func BuildInfo() string { + version := Version + if info, ok := debug.ReadBuildInfo(); ok && info.Main.Version != "(devel)" { + version = info.Main.Version + } + + return fmt.Sprintf(`%s %s/%s %s + +default config: %s +default state_dir: %s +default runtime_dir: %s`, + version, runtime.GOOS, runtime.GOARCH, runtime.Version(), + filepath.Join(ConfigDirectory, "maddy.conf"), + DefaultStateDirectory, + DefaultRuntimeDirectory) +} + +func init() { + maddycli.AddGlobalFlag( + &cli.PathFlag{ + Name: "config", + Usage: "Configuration file to use", + EnvVars: []string{"MADDY_CONFIG"}, + Value: filepath.Join(ConfigDirectory, "maddy.conf"), + }, + ) + maddycli.AddGlobalFlag(&cli.BoolFlag{ + Name: "debug", + Usage: "enable debug logging early", + Destination: &log.DefaultLogger.Debug, + }) + maddycli.AddSubcommand(&cli.Command{ + Name: "run", + Usage: "Start the server", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "libexec", + Value: DefaultLibexecDirectory, + Usage: "path to the libexec directory", + Destination: &config.LibexecDirectory, + }, + &cli.StringSliceFlag{ + Name: "log", + Usage: "default logging target(s)", + Value: cli.NewStringSlice("stderr"), + }, + &cli.BoolFlag{ + Name: "v", + Usage: "print version and build metadata, then exit", + Hidden: true, + }, + }, + Action: Run, + }) + maddycli.AddSubcommand(&cli.Command{ + Name: "version", + Usage: "Print version and build metadata, then exit", + Action: func(c *cli.Context) error { + fmt.Println(BuildInfo()) + return nil + }, + }) + + if enableDebugFlags { + maddycli.AddGlobalFlag(&cli.StringFlag{ + Name: "debug.pprof", + Usage: "enable live profiler HTTP endpoint and listen on the specified address", + }) + maddycli.AddGlobalFlag(&cli.IntFlag{ + Name: "debug.blockprofrate", + Usage: "set blocking profile rate", + }) + maddycli.AddGlobalFlag(&cli.IntFlag{ + Name: "debug.mutexproffract", + Usage: "set mutex profile fraction", + }) + } +} + +// Run is the entry point for all server-running code. It takes care of command line arguments processing, +// logging initialization, directives setup, configuration reading. After all that, it +// calls moduleMain to initialize and run modules. +func Run(c *cli.Context) error { + certmagic.UserAgent = "maddy/" + Version + + if c.NArg() != 0 { + return cli.Exit(fmt.Sprintln("usage:", os.Args[0], "[options]"), 2) + } + + if c.Bool("v") { + fmt.Println("maddy", BuildInfo()) + return nil + } + + var err error + log.DefaultLogger.Out, err = LogOutputOption(c.StringSlice("log")) + if err != nil { + systemdStatusErr(err) + return cli.Exit(err.Error(), 2) + } + + initDebug(c) + + os.Setenv("PATH", config.LibexecDirectory+string(filepath.ListSeparator)+os.Getenv("PATH")) + + f, err := os.Open(c.Path("config")) + if err != nil { + systemdStatusErr(err) + return cli.Exit(err.Error(), 2) + } + defer f.Close() + + cfg, err := parser.Read(f, c.Path("config")) + if err != nil { + systemdStatusErr(err) + return cli.Exit(err.Error(), 2) + } + + defer log.DefaultLogger.Out.Close() + + if err := moduleMain(cfg); err != nil { + systemdStatusErr(err) + return cli.Exit(err.Error(), 1) + } + + return nil +} + +func initDebug(c *cli.Context) { + if !enableDebugFlags { + return + } + + if c.IsSet("debug.pprof") { + profileEndpoint := c.String("debug.pprof") + go func() { + log.Println("listening on", "http://"+profileEndpoint, "for profiler requests") + log.Println("failed to listen on profiler endpoint:", http.ListenAndServe(profileEndpoint, nil)) + }() + } + + // These values can also be affected by environment so set them + // only if argument is specified. + if c.IsSet("debug.mutexproffract") { + runtime.SetMutexProfileFraction(c.Int("debug.mutexproffract")) + } + if c.IsSet("debug.blockprofrate") { + runtime.SetBlockProfileRate(c.Int("debug.blockprofrate")) + } +} + +func InitDirs() error { + if config.StateDirectory == "" { + config.StateDirectory = DefaultStateDirectory + } + if config.RuntimeDirectory == "" { + config.RuntimeDirectory = DefaultRuntimeDirectory + } + if config.LibexecDirectory == "" { + config.LibexecDirectory = DefaultLibexecDirectory + } + + if err := ensureDirectoryWritable(config.StateDirectory); err != nil { + return err + } + if err := ensureDirectoryWritable(config.RuntimeDirectory); err != nil { + return err + } + + // Make sure all paths we are going to use are absolute + // before we change the working directory. + if !filepath.IsAbs(config.StateDirectory) { + return errors.New("statedir should be absolute") + } + if !filepath.IsAbs(config.RuntimeDirectory) { + return errors.New("runtimedir should be absolute") + } + if !filepath.IsAbs(config.LibexecDirectory) { + return errors.New("-libexec should be absolute") + } + + // Change the working directory to make all relative paths + // in configuration relative to state directory. + if err := os.Chdir(config.StateDirectory); err != nil { + log.Println(err) + } + + return nil +} + +func ensureDirectoryWritable(path string) error { + if err := os.MkdirAll(path, 0o700); err != nil { + return err + } + + testFile, err := os.Create(filepath.Join(path, "writeable-test")) + if err != nil { + return err + } + testFile.Close() + return os.RemoveAll(testFile.Name()) +} + +func ReadGlobals(cfg []config.Node) (map[string]interface{}, []config.Node, error) { + globals := config.NewMap(nil, config.Node{Children: cfg}) + globals.String("state_dir", false, false, DefaultStateDirectory, &config.StateDirectory) + globals.String("runtime_dir", false, false, DefaultRuntimeDirectory, &config.RuntimeDirectory) + globals.String("hostname", false, false, "", nil) + globals.String("autogenerated_msg_domain", false, false, "", nil) + globals.Custom("tls", false, false, nil, tls.TLSDirective, nil) + globals.Custom("tls_client", false, false, nil, tls.TLSClientBlock, nil) + globals.Bool("storage_perdomain", false, false, nil) + globals.Bool("auth_perdomain", false, false, nil) + globals.StringList("auth_domains", false, false, nil, nil) + globals.Custom("log", false, false, defaultLogOutput, logOutput, &log.DefaultLogger.Out) + globals.Bool("debug", false, log.DefaultLogger.Debug, &log.DefaultLogger.Debug) + config.EnumMapped(globals, "auth_map_normalize", true, false, authz.NormalizeFuncs, authz.NormalizeAuto, nil) + modconfig.Table(globals, "auth_map", true, false, nil, nil) + globals.AllowUnknown() + unknown, err := globals.Process() + return globals.Values, unknown, err +} + +func moduleMain(cfg []config.Node) error { + globals, modBlocks, err := ReadGlobals(cfg) + if err != nil { + return err + } + + if err := InitDirs(); err != nil { + return err + } + + hooks.AddHook(hooks.EventLogRotate, reinitLogging) + + endpoints, mods, err := RegisterModules(globals, modBlocks) + if err != nil { + return err + } + + err = initModules(globals, endpoints, mods) + if err != nil { + return err + } + + systemdStatus(SDReady, "Listening for incoming connections...") + + handleSignals() + + systemdStatus(SDStopping, "Waiting for running transactions to complete...") + + hooks.RunHooks(hooks.EventShutdown) + + return nil +} + +type ModInfo struct { + Instance module.Module + Cfg config.Node +} + +func RegisterModules(globals map[string]interface{}, nodes []config.Node) (endpoints, mods []ModInfo, err error) { + mods = make([]ModInfo, 0, len(nodes)) + + for _, block := range nodes { + var instName string + var modAliases []string + if len(block.Args) == 0 { + instName = block.Name + } else { + instName = block.Args[0] + modAliases = block.Args[1:] + } + + modName := block.Name + + endpFactory := module.GetEndpoint(modName) + if endpFactory != nil { + inst, err := endpFactory(modName, block.Args) + if err != nil { + return nil, nil, err + } + + endpoints = append(endpoints, ModInfo{Instance: inst, Cfg: block}) + continue + } + + factory := module.Get(modName) + if factory == nil { + return nil, nil, config.NodeErr(block, "unknown module or global directive: %s", modName) + } + + if module.HasInstance(instName) { + return nil, nil, config.NodeErr(block, "config block named %s already exists", instName) + } + + inst, err := factory(modName, instName, modAliases, nil) + if err != nil { + return nil, nil, err + } + + module.RegisterInstance(inst, config.NewMap(globals, block)) + for _, alias := range modAliases { + if module.HasInstance(alias) { + return nil, nil, config.NodeErr(block, "config block named %s already exists", alias) + } + module.RegisterAlias(alias, instName) + } + + log.Debugf("%v:%v: register config block %v %v", block.File, block.Line, instName, modAliases) + mods = append(mods, ModInfo{Instance: inst, Cfg: block}) + } + + if len(endpoints) == 0 { + return nil, nil, fmt.Errorf("at least one endpoint should be configured") + } + + return endpoints, mods, nil +} + +func initModules(globals map[string]interface{}, endpoints, mods []ModInfo) error { + for _, endp := range endpoints { + if err := endp.Instance.Init(config.NewMap(globals, endp.Cfg)); err != nil { + return err + } + + if closer, ok := endp.Instance.(io.Closer); ok { + endp := endp + hooks.AddHook(hooks.EventShutdown, func() { + log.Debugf("close %s (%s)", endp.Instance.Name(), endp.Instance.InstanceName()) + if err := closer.Close(); err != nil { + log.Printf("module %s (%s) close failed: %v", endp.Instance.Name(), endp.Instance.InstanceName(), err) + } + }) + } + } + + for _, inst := range mods { + if module.Initialized[inst.Instance.InstanceName()] { + continue + } + + return fmt.Errorf("Unused configuration block at %s:%d - %s (%s)", + inst.Cfg.File, inst.Cfg.Line, inst.Instance.InstanceName(), inst.Instance.Name()) + } + + return nil +} diff --git a/maddy_debug.go b/maddy_debug.go new file mode 100644 index 0000000..dc03168 --- /dev/null +++ b/maddy_debug.go @@ -0,0 +1,30 @@ +//go:build debugflags +// +build debugflags + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package maddy + +import ( + _ "net/http/pprof" +) + +func init() { + enableDebugFlags = true +} diff --git a/signal.go b/signal.go new file mode 100644 index 0000000..e952fcf --- /dev/null +++ b/signal.go @@ -0,0 +1,66 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package maddy + +import ( + "os" + "os/signal" + "syscall" + + "github.com/foxcpp/maddy/framework/hooks" + "github.com/foxcpp/maddy/framework/log" +) + +// handleSignals function creates and listens on OS signals channel. +// +// OS-specific signals that correspond to the program termination +// (SIGTERM, SIGHUP, SIGINT) will cause this function to return. +// +// SIGUSR1 will call reinitLogging without returning. +func handleSignals() os.Signal { + sig := make(chan os.Signal, 5) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGINT, syscall.SIGUSR1, syscall.SIGUSR2) + + for { + switch s := <-sig; s { + case syscall.SIGUSR1: + log.Printf("signal received (%s), rotating logs", s.String()) + systemdStatus(SDReloading, "Reopening logs...") + hooks.RunHooks(hooks.EventLogRotate) + systemdStatus(SDReady, "Listening for incoming connections...") + case syscall.SIGUSR2: + log.Printf("signal received (%s), reloading state", s.String()) + systemdStatus(SDReloading, "Reloading state...") + hooks.RunHooks(hooks.EventReload) + systemdStatus(SDReady, "Listening for incoming connections...") + default: + go func() { + s := handleSignals() + log.Printf("forced shutdown due to signal (%v)!", s) + os.Exit(1) + }() + + log.Printf("signal received (%v), next signal will force immediate shutdown.", s) + return s + } + } +} diff --git a/signal_nonposix.go b/signal_nonposix.go new file mode 100644 index 0000000..87ea0cd --- /dev/null +++ b/signal_nonposix.go @@ -0,0 +1,45 @@ +//go:build windows || plan9 +// +build windows plan9 + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package maddy + +import ( + "os" + "os/signal" + "syscall" + + "github.com/foxcpp/maddy/framework/log" +) + +func handleSignals() os.Signal { + sig := make(chan os.Signal, 5) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGINT) + + s := <-sig + go func() { + s := handleSignals() + log.Printf("forced shutdown due to signal (%v)!", s) + os.Exit(1) + }() + + log.Printf("signal received (%v), next signal will force immediate shutdown.", s) + return s +} diff --git a/systemd.go b/systemd.go new file mode 100644 index 0000000..649740d --- /dev/null +++ b/systemd.go @@ -0,0 +1,133 @@ +//go:build linux +// +build linux + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package maddy + +import ( + "errors" + "fmt" + "io" + "net" + "os" + "strings" + "syscall" + + "github.com/foxcpp/maddy/framework/log" +) + +type SDStatus string + +const ( + SDReady = "READY=1" + SDReloading = "RELOADING=1" + SDStopping = "STOPPING=1" +) + +var ErrNoNotifySock = errors.New("no systemd socket") + +func sdNotifySock() (*net.UnixConn, error) { + sockAddr := os.Getenv("NOTIFY_SOCKET") + if sockAddr == "" { + return nil, ErrNoNotifySock + } + if strings.HasPrefix(sockAddr, "@") { + sockAddr = "\x00" + sockAddr[1:] + } + + return net.DialUnix("unixgram", nil, &net.UnixAddr{ + Name: sockAddr, + Net: "unixgram", + }) +} + +func setScmPassCred(sock *net.UnixConn) error { + sConn, err := sock.SyscallConn() + if err != nil { + return err + } + + var sockoptErr error + if err := sConn.Control(func(fd uintptr) { + sockoptErr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_PASSCRED, 1) + }); err != nil { + return err + } + if sockoptErr != nil { + return sockoptErr + } + return nil +} + +func systemdStatus(status SDStatus, desc string) { + sock, err := sdNotifySock() + if err != nil { + if !errors.Is(err, ErrNoNotifySock) { + log.Println("systemd: failed to acquire notify socket:", err) + } + return + } + defer sock.Close() + + if err := setScmPassCred(sock); err != nil { + log.Println("systemd: failed to set SCM_PASSCRED on the socket:", err) + } + + if desc != "" { + if _, err := io.WriteString(sock, fmt.Sprintf("%s\nSTATUS=%s", status, desc)); err != nil { + log.Println("systemd: I/O error:", err) + } + log.Debugf(`systemd: %s STATUS="%s"`, status, desc) + } else { + if _, err := io.WriteString(sock, string(status)); err != nil { + log.Println("systemd: I/O error:", err) + } + log.Debugf(`systemd: %s`, status) + } +} + +func systemdStatusErr(reportedErr error) { + sock, err := sdNotifySock() + if err != nil { + if !errors.Is(err, ErrNoNotifySock) { + log.Println("systemd: failed to acquire notify socket:", err) + } + return + } + defer sock.Close() + + if err := setScmPassCred(sock); err != nil { + log.Println("systemd: failed to set SCM_PASSCRED on the socket:", err) + } + + var errno syscall.Errno + if errors.As(reportedErr, &errno) { + log.Debugf(`systemd: ERRNO=%d STATUS="%v"`, errno, reportedErr) + if _, err := io.WriteString(sock, fmt.Sprintf("ERRNO=%d\nSTATUS=%v", errno, reportedErr)); err != nil { + log.Println("systemd: I/O error:", err) + } + return + } + + if _, err := io.WriteString(sock, fmt.Sprintf("STATUS=%v\n", reportedErr)); err != nil { + log.Println("systemd: I/O error:", err) + } + log.Debugf(`systemd: STATUS="%v"`, reportedErr) +} diff --git a/systemd_nonlinux.go b/systemd_nonlinux.go new file mode 100644 index 0000000..e31cd15 --- /dev/null +++ b/systemd_nonlinux.go @@ -0,0 +1,34 @@ +//go:build !linux +// +build !linux + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package maddy + +type SDStatus string + +const ( + SDReady = "READY=1" + SDReloading = "RELOADING=1" + SDStopping = "STOPPING=1" +) + +func systemdStatus(SDStatus, string) {} + +func systemdStatusErr(error) {} diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..765a7b4 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,17 @@ +# maddy integration testing + +## Tests structure + +The test library creates a temporary state and runtime directory, starts the +server with the specified configuration file and lets you interact with it +using a couple of convenient wrappers. + +## Running + +To run tests, use `go test -tags integration` in this directory. Make sure to +have a maddy executable in the current working directory. +Use `-integration.executable` if the executable is named different or is placed +somewhere else. +Use `-integration.coverprofile` to pass `-test.coverprofile +your_value.RANDOM` to test executable. See `./build_cover.sh` to build a +server executable instrumented with coverage counters. diff --git a/tests/basic_test.go b/tests/basic_test.go new file mode 100644 index 0000000..80cf1ae --- /dev/null +++ b/tests/basic_test.go @@ -0,0 +1,63 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "testing" + + "github.com/foxcpp/maddy/tests" +) + +func TestBasic(tt *testing.T) { + tt.Parallel() + + // This test is mostly intended to test whether the integration testing + // library is working as expected. + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.ExpectPattern("220 mx.maddy.test *") + conn.Writeln("EHLO localhost") + conn.ExpectPattern("250-*") + conn.ExpectPattern("250-PIPELINING") + conn.ExpectPattern("250-8BITMIME") + conn.ExpectPattern("250-ENHANCEDSTATUSCODES") + conn.ExpectPattern("250-CHUNKING") + conn.ExpectPattern("250-SMTPUTF8") + conn.ExpectPattern("250-SIZE *") + conn.ExpectPattern("250 LIMITS RCPTMAX=20000") + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} diff --git a/tests/build_cover.sh b/tests/build_cover.sh new file mode 100755 index 0000000..929511c --- /dev/null +++ b/tests/build_cover.sh @@ -0,0 +1,5 @@ +#!/bin/sh +if [ -z "$GO" ]; then + GO=go +fi +exec $GO test -tags 'cover_main debugflags' -coverpkg 'github.com/foxcpp/maddy,github.com/foxcpp/maddy/pkg/...,github.com/foxcpp/maddy/internal/...' -cover -covermode atomic -c cover_test.go -o maddy.cover diff --git a/tests/conn.go b/tests/conn.go new file mode 100644 index 0000000..9ff86e8 --- /dev/null +++ b/tests/conn.go @@ -0,0 +1,289 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests + +import ( + "bufio" + "crypto/tls" + "encoding/base64" + "fmt" + "io" + "net" + "path" + "strconv" + "strings" + "time" +) + +// Conn is a helper that simplifies testing of text protocol interactions. +type Conn struct { + T *T + + WriteTimeout time.Duration + ReadTimeout time.Duration + + allowIOErr bool + + Conn net.Conn + Scanner *bufio.Scanner +} + +// AllowIOErr toggles whether I/O errors should be returned to the caller of +// Conn method or should immedately fail the test. +// +// By default (ok = false), the latter happens. +func (c *Conn) AllowIOErr(ok bool) { + c.allowIOErr = ok +} + +// Write writes the string to the connection socket. +func (c *Conn) Write(s string) { + c.T.Helper() + + // Make sure the test will not accidentally hang waiting for I/O forever if + // the server breaks. + if err := c.Conn.SetWriteDeadline(time.Now().Add(c.WriteTimeout)); err != nil { + c.fatal("Cannot set write deadline: %v", err) + } + defer func() { + if err := c.Conn.SetWriteDeadline(time.Time{}); err != nil { + c.log('-', "Failed to reset connection deadline: %v", err) + } + }() + + c.log('>', "%s", s) + if _, err := io.WriteString(c.Conn, s); err != nil { + c.fatal("Unexpected I/O error: %v", err) + } +} + +func (c *Conn) Writeln(s string) { + c.T.Helper() + + c.Write(s + "\r\n") +} + +func (c *Conn) Readln() (string, error) { + c.T.Helper() + + // Make sure the test will not accidentally hang waiting for I/O forever if + // the server breaks. + if err := c.Conn.SetReadDeadline(time.Now().Add(c.ReadTimeout)); err != nil { + c.fatal("Cannot set write deadline: %v", err) + } + defer func() { + if err := c.Conn.SetReadDeadline(time.Time{}); err != nil { + c.log('-', "Failed to reset connection deadline: %v", err) + } + }() + + if !c.Scanner.Scan() { + if err := c.Scanner.Err(); err != nil { + if c.allowIOErr { + return "", err + } + c.fatal("Unexpected I/O error: %v", err) + } + if c.allowIOErr { + return "", io.EOF + } + c.fatal("Unexpected EOF") + } + + c.log('<', "%v", c.Scanner.Text()) + + return c.Scanner.Text(), nil +} + +func (c *Conn) Expect(line string) { + c.T.Helper() + + actual, err := c.Readln() + if err != nil { + c.T.Fatal("Unexpected I/O error:", err) + } + + if line != actual { + c.T.Fatalf("Response line not matching the expected one, want %q", line) + } +} + +// ExpectPattern reads a line from the connection socket and checks whether is +// matches the supplied shell pattern (as defined by path.Match). The original +// line is returned. +func (c *Conn) ExpectPattern(pat string) string { + c.T.Helper() + + line, err := c.Readln() + if err != nil { + c.T.Fatal("Unexpected I/O error:", err) + } + + match, err := path.Match(pat, line) + if err != nil { + c.T.Fatal("Malformed pattern:", err) + } + if !match { + c.T.Fatalf("Response line not matching the expected pattern, want %q", pat) + } + + return line +} + +func (c *Conn) fatal(f string, args ...interface{}) { + c.T.Helper() + c.log('-', f, args...) + c.T.FailNow() +} + +func (c *Conn) log(direction rune, f string, args ...interface{}) { + c.T.Helper() + + local, remote := c.Conn.LocalAddr().(*net.TCPAddr), c.Conn.RemoteAddr().(*net.TCPAddr) + msg := strings.Builder{} + if local.IP.IsLoopback() { + msg.WriteString(strconv.Itoa(local.Port)) + } else { + msg.WriteString(local.String()) + } + + msg.WriteRune(' ') + msg.WriteRune(direction) + msg.WriteRune(' ') + + if remote.IP.IsLoopback() { + textPort := c.T.portsRev[uint16(remote.Port)] + if textPort != "" { + msg.WriteString(textPort) + } else { + msg.WriteString(strconv.Itoa(remote.Port)) + } + } else { + msg.WriteString(local.String()) + } + + if _, ok := c.Conn.(*tls.Conn); ok { + msg.WriteString(" [tls]") + } + msg.WriteString(": ") + fmt.Fprintf(&msg, f, args...) + c.T.Log(strings.TrimRight(msg.String(), "\r\n ")) +} + +func (c *Conn) TLS() { + c.T.Helper() + + tlsC := tls.Client(c.Conn, &tls.Config{ + ServerName: "maddy.test", + InsecureSkipVerify: true, + }) + if err := tlsC.Handshake(); err != nil { + c.fatal("TLS handshake fail: %v", err) + } + + c.Conn = tlsC + c.Scanner = bufio.NewScanner(c.Conn) +} + +func (c *Conn) SMTPPlainAuth(username, password string, expectOk bool) { + c.T.Helper() + + resp := append([]byte{0x00}, username...) + resp = append(resp, 0x00) + resp = append(resp, password...) + c.Writeln("AUTH PLAIN " + base64.StdEncoding.EncodeToString(resp)) + if expectOk { + c.ExpectPattern("235 *") + } else { + c.ExpectPattern("*") + } +} + +func (c *Conn) SMTPNegotation(ourName string, requireExts, blacklistExts []string) { + c.T.Helper() + + needCapsMap := make(map[string]bool) + blacklistCapsMap := make(map[string]bool) + for _, ext := range requireExts { + needCapsMap[ext] = false + } + for _, ext := range blacklistExts { + blacklistCapsMap[ext] = false + } + + c.Writeln("EHLO " + ourName) + + // Consume the first line from socket, it is either initial greeting (sent + // before we sent EHLO) or the EHLO reply in case of re-negotiation after + // STARTTLS. + l, err := c.Readln() + if err != nil { + c.T.Fatal("I/O error during SMTP negotiation:", err) + } + if strings.HasPrefix(l, "220") { + // That was initial greeting, consume one more line. + c.ExpectPattern("250-*") + } + + var caps []string +capsloop: + for { + line, err := c.Readln() + if err != nil { + c.T.Fatal("I/O error during SMTP negotiation:", err) + } + + switch { + case strings.HasPrefix(line, "250-"): + caps = append(caps, strings.TrimPrefix(line, "250-")) + case strings.HasPrefix(line, "250 "): + caps = append(caps, strings.TrimPrefix(line, "250 ")) + break capsloop + default: + c.T.Fatal("Unexpected reply during SMTP negotiation:", line) + } + } + + for _, ext := range caps { + needCapsMap[ext] = true + if _, ok := blacklistCapsMap[ext]; ok { + blacklistCapsMap[ext] = true + } + } + for ext, status := range needCapsMap { + if !status { + c.T.Fatalf("Capability %v is missing but required", ext) + } + } + for ext, status := range blacklistCapsMap { + if status { + c.T.Fatalf("Capability %v is present but not allowed", ext) + } + } +} + +func (c *Conn) Close() error { + return c.Conn.Close() +} + +func (c *Conn) Rebind(subtest *T) *Conn { + cpy := *c + cpy.T = subtest + return &cpy +} diff --git a/tests/cover_test.go b/tests/cover_test.go new file mode 100644 index 0000000..298e3e9 --- /dev/null +++ b/tests/cover_test.go @@ -0,0 +1,91 @@ +//go:build cover_main +// +build cover_main + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests + +/* +Go toolchain lacks the ability to instrument arbitrary executables with +coverage counters. + +This file wraps the maddy executable into a minimal layer of "test" logic to +make 'go test' work for it and produce the coverage report. + +Use ./build_cover.sh to compile it into ./maddy.cover. + +References: +https://stackoverflow.com/questions/43381335/how-to-capture-code-coverage-from-a-go-binary +https://blog.cloudflare.com/go-coverage-with-external-tests/ +https://github.com/albertito/chasquid/blob/master/coverage_test.go +*/ + +import ( + "flag" + "io" + "os" + "testing" + + _ "github.com/foxcpp/maddy" // To register run command + _ "github.com/foxcpp/maddy/internal/cli/ctl" // To register other CLI commands. + + maddycli "github.com/foxcpp/maddy/internal/cli" +) + +func TestMain(m *testing.M) { + // -test.* flags are registered somewhere in init() in "testing" (?). + + // maddy.Run changes the working directory, we need to change it back so + // -test.coverprofile writes out profile in the right location. + wd, err := os.Getwd() + if err != nil { + panic(err) + } + + // Skip flag parsing and make flag.Parse no-op so when + // m.Run calls it it will not error out on maddy flags. + args := os.Args + os.Args = []string{"command"} + flag.Parse() + os.Args = args + + code := maddycli.RunWithoutExit() + + if err := os.Chdir(wd); err != nil { + panic(err) + } + + // Silence output produced by "testing" runtime. + r, w, err := os.Pipe() + if err == nil { + os.Stderr = w + os.Stdout = w + } + go func() { + _, _ = io.ReadAll(r) + }() + + // Even though we do not have any tests to run, we need to call out into + // "testing" to make it process flags and produce the coverage report. + m.Run() + + // TestMain doc says we have to exit with a sensible status code on our + // own. + os.Exit(code) +} diff --git a/tests/dovecot_sasl_test.go b/tests/dovecot_sasl_test.go new file mode 100644 index 0000000..af424e8 --- /dev/null +++ b/tests/dovecot_sasl_test.go @@ -0,0 +1,206 @@ +//go:build integration && (darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris) +// +build integration +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// only posix systems ^ + +package tests_test + +import ( + "bufio" + "errors" + "flag" + "io/ioutil" + "os" + "os/exec" + "os/user" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/foxcpp/maddy/tests" +) + +var DovecotExecutable string + +func init() { + flag.StringVar(&DovecotExecutable, "integration.dovecot", "dovecot", "path to dovecot executable for interop tests") +} + +const dovecotConf = `base_dir = $ROOT/run/ +state_dir = $ROOT/lib/ +log_path = /dev/stderr +ssl = no + +default_internal_user = $USER +default_internal_group = $GROUP +default_login_user = $USER + +passdb { + driver = passwd-file + args = $ROOT/passwd +} + +userdb { + driver = passwd-file + args = $ROOT/passwd +} + +service auth { + unix_listener auth { + mode = 0666 + } +} + +# Dovecot refuses to start without protocols, so we need to give it one. +protocols = imap + +service imap-login { + chroot = + inet_listener imap { + address = 127.0.0.1 + port = 0 + } +} + +service anvil { + chroot = +} + +# Turn on debugging information, to help troubleshooting issues. +auth_verbose = yes +auth_debug = yes +auth_debug_passwords = yes +auth_verbose_passwords = yes +mail_debug = yes +` + +const dovecotPasswd = `tester:{plain}123456:1000:1000::/home/user` + +func runDovecot(t *testing.T) (string, *exec.Cmd) { + dovecotExec, err := exec.LookPath(DovecotExecutable) + if err != nil { + if errors.Is(err, exec.ErrNotFound) { + t.Skip("No Dovecot executable found, skipping interop. tests") + } + t.Fatal(err) + } + + tempDir := t.TempDir() + + curUser, err := user.Current() + if err != nil { + t.Fatal(err) + } + curGroup, err := user.LookupGroupId(curUser.Gid) + if err != nil { + t.Fatal(err) + } + + dovecotConf := strings.NewReplacer( + "$ROOT", tempDir, + "$USER", curUser.Username, + "$GROUP", curGroup.Name).Replace(dovecotConf) + err = ioutil.WriteFile(filepath.Join(tempDir, "dovecot.conf"), []byte(dovecotConf), os.ModePerm) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(tempDir, "passwd"), []byte(dovecotPasswd), os.ModePerm) + if err != nil { + t.Fatal(err) + } + + cmd := exec.Command(dovecotExec, "-F", "-c", filepath.Join(tempDir, "dovecot.conf")) + stderr, err := cmd.StderrPipe() + if err != nil { + t.Fatal(err) + } + + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + + ready := make(chan struct{}, 1) + + go func() { + scnr := bufio.NewScanner(stderr) + for scnr.Scan() { + line := scnr.Text() + + // One of messages printed near completing initialization. + if strings.Contains(line, "starting up for imap") { + time.Sleep(500*time.Millisecond) + ready <- struct{}{} + } + + t.Log("dovecot:", line) + } + if err := scnr.Err(); err != nil { + t.Log("stderr I/O error:", err) + } + }() + + <-ready + + return tempDir, cmd +} + +func cleanDovecot(t *testing.T, tempDir string, cmd *exec.Cmd) { + cmd.Process.Signal(syscall.SIGTERM) + if !t.Failed() { + os.RemoveAll(tempDir) + } else { + t.Log("Dovecot directory is not deleted:", tempDir) + } +} + +func TestDovecotSASLClient(tt *testing.T) { + tt.Parallel() + + dovecotDir, cmd := runDovecot(tt) + defer cleanDovecot(tt, dovecotDir, cmd) + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Env("DOVECOT_SASL_SOCK=" + filepath.Join(dovecotDir, "run", "auth-client")) + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + auth dovecot_sasl unix://{env:DOVECOT_SASL_SOCK} + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("localhost", nil, nil) + c.Writeln("AUTH PLAIN AHRlc3QAMTIzNDU2") // 0x00 test 0x00 123456 (invalid user) + c.ExpectPattern("535 *") + c.Writeln("AUTH PLAIN AHRlc3RlcgAxMjM0NQ==") // 0x00 tester 0x00 12345 (invalid password) + c.ExpectPattern("535 *") + c.Writeln("AUTH PLAIN AHRlc3RlcgAxMjM0NTY=") // 0x00 tester 0x00 123456 + c.ExpectPattern("235 *") +} diff --git a/tests/dovecot_sasld_test.go b/tests/dovecot_sasld_test.go new file mode 100644 index 0000000..8fa2fd6 --- /dev/null +++ b/tests/dovecot_sasld_test.go @@ -0,0 +1,202 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "bufio" + "errors" + "flag" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/foxcpp/maddy/tests" +) + +var ChasquidExecutable string + +func init() { + flag.StringVar(&ChasquidExecutable, "integration.chasquid", "chasquid", "path to chasquid executable for interop tests") +} + +const chasquidConf = `smtp_address: "127.0.0.2:44444" +submission_address: "127.0.0.1:44443" + +data_dir: "$ROOT" +mail_log_path: "/dev/null" + +dovecot_auth: true +dovecot_userdb_path: "$AUTH_CLIENT" # needs any Unix socket, not actually used +dovecot_client_path: "$AUTH_CLIENT" +` + +// RSA 1024, valid for *.example.invalid, 127.0.0.1, 127.0.0.2,, 127.0.0.3 +// until Nov 18 17:13:45 2029 GMT. +const testServerCert = `-----BEGIN CERTIFICATE----- +MIICDzCCAXigAwIBAgIRAJ1x+qCW7L+Hs6sRU8BHmWkwDQYJKoZIhvcNAQELBQAw +EjEQMA4GA1UEChMHQWNtZSBDbzAeFw0xOTExMTgxNzEzNDVaFw0yOTExMTUxNzEz +NDVaMBIxEDAOBgNVBAoTB0FjbWUgQ28wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ +AoGBAPINKMyuu3AvzndLDS2/BroA+DRUcAhWPBxMxG1b1BkkHisAZWteKajKmwdO +O13N8HHBRPPOD56AAPLZGNxYLHn6nel7AiH8k40/xC5tDOthqA82+00fwJHDFCnW +oDLOLcO17HulPvfCSWfefc+uee4kajPa+47hutqZH2bGMTXhAgMBAAGjZTBjMA4G +A1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAA +MC4GA1UdEQQnMCWCESouZXhhbXBsZS5pbnZhbGlkhwR/AAABhwR/AAAChwR/AAAD +MA0GCSqGSIb3DQEBCwUAA4GBAGRn3C2NbwR4cyQmTRm5jcaqi1kAYyEu6U8Q9PJW +Q15BXMKUTx2lw//QScK9MH2JpKxDuzWDSvaxZMnTxgri2uiplqpe8ydsWj6Wl0q9 +2XMGJ9LIxTZk5+cyZP2uOolvmSP/q8VFTyk9Udl6KUZPQyoiiDq4rBFUIxUyb+bX +pHkR +-----END CERTIFICATE-----` + +const testServerKey = `-----BEGIN PRIVATE KEY----- +MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAPINKMyuu3AvzndL +DS2/BroA+DRUcAhWPBxMxG1b1BkkHisAZWteKajKmwdOO13N8HHBRPPOD56AAPLZ +GNxYLHn6nel7AiH8k40/xC5tDOthqA82+00fwJHDFCnWoDLOLcO17HulPvfCSWfe +fc+uee4kajPa+47hutqZH2bGMTXhAgMBAAECgYEAgPjSDH3uEdDnSlkLJJzskJ+D +oR58s3R/gvTElSCg2uSLzo3ffF4oBHAwOqxMpabdvz8j5mSdne7Gkp9qx72TtEG2 +wt6uX1tZhm2UTAkInH8IQDthj98P8vAWQsS6HHEIMErsrW2CyUrAt/+o1BRg/hWW +zixA3CLTthhZTJkaUCECQQD5EM16UcTAKfhr3IZppgq+ZsAOMkeCl3XVV9gHo32i +DL6UFAb27BAYyjfcZB1fPou4RszX0Ryu9yU0P5qm6N47AkEA+MpdAPkaPziY0ok4 +e9Tcee6P0mIR+/AHk9GliVX2P74DDoOHyMXOSRBwdb+z2tYjrdjkNEL1Txe+sHny +k/EukwJBAOBqlmqPwNNRPeiaRHZvSSD0XjqsbSirJl48D4gadPoNt66fOQNGAt8D +Xj/z6U9HgQdiq/IOFmVEhT5FzSh1jL8CQQD3Myth8iGQO84tM0c6U3CWfuHMqsEv +0XnV+HNAmHdLMqOa4joi1dh4ZKs5dDdi828UJ/PnsbhI1FEWzLSpJvWdAkAkVWqf +AC/TvWvEZLA6Z5CllyNzZJ7XvtIaNOosxHDolyZ1HMWMlfEb2K2ZXWLy5foKPeoY +Xi3olS9rB0J+Rvjz +-----END PRIVATE KEY-----` + +func runChasquid(t *testing.T, authClientPath string) (string, *exec.Cmd) { + tempDir := t.TempDir() + t.Log("Using", tempDir) + + chasquidConf := strings.NewReplacer( + "$ROOT", tempDir, + "$AUTH_CLIENT", authClientPath).Replace(chasquidConf) + err := ioutil.WriteFile(filepath.Join(tempDir, "chasquid.conf"), []byte(chasquidConf), os.ModePerm) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(tempDir, "certs", "example.org"), os.ModePerm); err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(tempDir, "certs", "example.org", "fullchain.pem"), []byte(testServerCert), os.ModePerm) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(tempDir, "certs", "example.org", "privkey.pem"), []byte(testServerKey), os.ModePerm) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(tempDir, "domains", "example.org"), os.ModePerm); err != nil { + t.Fatal(err) + } + + err = ioutil.WriteFile(filepath.Join(tempDir, "chasquid.conf"), []byte(chasquidConf), os.ModePerm) + if err != nil { + t.Fatal(err) + } + + cmd := exec.Command(ChasquidExecutable, "-v=2", "-config_dir", tempDir) + t.Log("Launching", cmd.String()) + stderr, err := cmd.StderrPipe() + if err != nil { + t.Fatal(err) + } + + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + + ready := make(chan struct{}, 1) + + go func() { + scnr := bufio.NewScanner(stderr) + for scnr.Scan() { + line := scnr.Text() + + // One of messages printed near completing initialization. + if strings.Contains(line, "Loading certificates") { + time.Sleep(1 * time.Second) + ready <- struct{}{} + } + + t.Log("chasquid:", line) + } + if err := scnr.Err(); err != nil { + t.Log("stderr I/O error:", err) + } + }() + + <-ready + + return tempDir, cmd +} + +func cleanChasquid(t *testing.T, tempDir string, cmd *exec.Cmd) { + cmd.Process.Signal(syscall.SIGTERM) + os.RemoveAll(tempDir) +} + +func TestSASLServerWithChasquid(tt *testing.T) { + tt.Parallel() + + _, err := exec.LookPath(ChasquidExecutable) + if err != nil { + if errors.Is(err, exec.ErrNotFound) { + tt.Skip("No chasquid executable found, skipping interop. tests") + } + tt.Fatal(err) + } + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + dovecot_sasld unix://{env:TEST_STATE_DIR}/auth.sock { + auth pass_table static { + # tester@example.org:123456 + entry tester@example.org "bcrypt:$2a$04$0SaXE/WOMBOfk5jyaKjo.OHkioRljdhMznLnYCg1nrksu9iLd51Ri" + } + }`) + t.Run(1) + defer t.Close() + + chasquidDir, cmd := runChasquid(tt, filepath.Join(t.StateDir(), "auth.sock")) + defer cleanChasquid(tt, chasquidDir, cmd) + + c := t.ConnUnnamed(44443) + defer c.Close() + c.SMTPNegotation("localhost", nil, nil) + c.Writeln("STARTTLS") + c.ExpectPattern("220 *") + c.TLS() + c.Writeln("AUTH PLAIN AHRlc3RAZXhhbXBsZS5vcmcAMTIzNDU2") // 0x00 test@example.org 0x00 123456 (invalid user) + c.ExpectPattern("535 *") + c.Writeln("AUTH PLAIN AHRlc3RlckBleGFtcGxlLm9yZwAxMjM0NQ==") // 0x00 tester 0x00 12345 (invalid password) + c.ExpectPattern("535 *") + c.Writeln("AUTH PLAIN AHRlc3RlckBleGFtcGxlLm9yZwAxMjM0NTY=") // 0x00 tester 0x00 123456 + c.ExpectPattern("235 *") +} diff --git a/tests/gocovcat.go b/tests/gocovcat.go new file mode 100644 index 0000000..4018912 --- /dev/null +++ b/tests/gocovcat.go @@ -0,0 +1,92 @@ +//usr/bin/env go run "$0" "$@"; exit $? +// +// From: https://git.lukeshu.com/go/cmd/gocovcat/ +// +//go:build ignore +// +build ignore + +// Copyright 2017 Luke Shumaker +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +// Command gocovcat combines multiple go cover runs, and prints the +// result on stdout. +package main + +import ( + "bufio" + "fmt" + "os" + "sort" + "strconv" + "strings" +) + +func handleErr(err error) { + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +func main() { + modeBool := false + blocks := map[string]int{} + for _, filename := range os.Args[1:] { + file, err := os.Open(filename) + handleErr(err) + buf := bufio.NewScanner(file) + for buf.Scan() { + line := buf.Text() + + if strings.HasPrefix(line, "mode: ") { + m := strings.TrimPrefix(line, "mode: ") + switch m { + case "set": + modeBool = true + case "count", "atomic": + // do nothing + default: + fmt.Fprintf(os.Stderr, "Unrecognized mode: %s\n", m) + os.Exit(1) + } + } else { + sp := strings.LastIndexByte(line, ' ') + block := line[:sp] + cntStr := line[sp+1:] + cnt, err := strconv.Atoi(cntStr) + handleErr(err) + blocks[block] += cnt + } + } + handleErr(buf.Err()) + } + keys := make([]string, 0, len(blocks)) + for key := range blocks { + keys = append(keys, key) + } + sort.Strings(keys) + modeStr := "count" + if modeBool { + modeStr = "set" + } + fmt.Printf("mode: %s\n", modeStr) + for _, block := range keys { + cnt := blocks[block] + if modeBool && cnt > 1 { + cnt = 1 + } + fmt.Printf("%s %d\n", block, cnt) + } +} diff --git a/tests/golangci-noisy.yml b/tests/golangci-noisy.yml new file mode 100644 index 0000000..d6ca6e0 --- /dev/null +++ b/tests/golangci-noisy.yml @@ -0,0 +1,23 @@ +linters: + enable: + - gosimple + - structcheck + - varcheck + - errcheck + - staticcheck + - ineffassign + - deadcode + - typecheck + - govet + - unused + - goimports + - prealloc + - unconvert + - misspell + - whitespace + - nakedret + - dogsled + - godox + - gocyclo + - dupl + - unparam diff --git a/tests/imap_test.go b/tests/imap_test.go new file mode 100644 index 0000000..ba3fabc --- /dev/null +++ b/tests/imap_test.go @@ -0,0 +1,96 @@ +//go:build integration && cgo && !nosqlite3 +// +build integration,cgo,!nosqlite3 + +package tests_test + +import ( + "testing" + + "github.com/foxcpp/maddy/tests" +) + +func TestIMAPEndpointAuthMap(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("imap") + t.Config(` + storage.imapsql test_store { + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth_map email_localpart + auth pass_table static { + entry "user" "bcrypt:$2a$10$E.AuCH3oYbaRrETXfXwc0.4jRAQBbanpZiCfudsJz9bHzLr/qj6ti" # password: 123 + } + storage &test_store + } + `) + t.Run(1) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN user@example.org 123") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) +} + +func TestIMAPEndpointStorageMap(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("imap") + t.Config(` + storage.imapsql test_store { + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + storage_map email_localpart + + auth_map email_localpart + auth pass_table static { + entry "user" "bcrypt:$2a$10$z9SvUwUjkY8wKOWd9IbISeEmbJua2cXRPqw7s2BnLXJuc6pIMPncK" # password: 123 + } + storage &test_store + } + `) + t.Run(1) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN user@example.org 123") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". CREATE testbox") + imapConn.ExpectPattern(". OK *") + + imapConn2 := t.Conn("imap") + defer imapConn2.Close() + imapConn2.ExpectPattern(`\* OK *`) + imapConn2.Writeln(". LOGIN user@example.com 123") + imapConn2.ExpectPattern(". OK *") + imapConn2.Writeln(`. LIST "" "*"`) + imapConn2.Expect(`* LIST (\HasNoChildren) "." INBOX`) + imapConn2.Expect(`* LIST (\HasNoChildren) "." "testbox"`) + imapConn2.ExpectPattern(". OK *") +} diff --git a/tests/imapsql_test.go b/tests/imapsql_test.go new file mode 100644 index 0000000..69f9e7e --- /dev/null +++ b/tests/imapsql_test.go @@ -0,0 +1,259 @@ +//go:build integration && cgo && !nosqlite3 +// +build integration,cgo,!nosqlite3 + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "testing" + "time" + + "github.com/foxcpp/maddy/tests" +) + +// Smoke test to ensure message delivery is handled correctly. + +func TestImapsqlDelivery(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("imap") + t.Port("smtp") + t.Config(` + storage.imapsql test_store { + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth dummy + storage &test_store + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + + deliver_to &test_store + } + `) + t.Run(2) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN testusr@maddy.test 1234") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) + + smtpConn := t.Conn("smtp") + defer smtpConn.Close() + smtpConn.SMTPNegotation("localhost", nil, nil) + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + smtpConn.Writeln("Hi!") + smtpConn.Writeln(".") + smtpConn.ExpectPattern("2*") + + time.Sleep(500 * time.Millisecond) + + imapConn.Writeln(". NOOP") + imapConn.ExpectPattern(`\* 1 EXISTS`) + imapConn.ExpectPattern(`\* 1 RECENT`) + imapConn.ExpectPattern(". OK *") + + imapConn.Writeln(". FETCH 1 (BODY.PEEK[])") + imapConn.ExpectPattern(`\* 1 FETCH (BODY\[\] {*}*`) + imapConn.Expect(`Delivered-To: testusr@maddy.test`) + imapConn.Expect(`Return-Path: `) + imapConn.ExpectPattern(`Received: from localhost (client.maddy.test \[` + tests.DefaultSourceIP.String() + `\]) by maddy.test`) + imapConn.ExpectPattern(` (envelope-sender ) with ESMTP id *; *`) + imapConn.ExpectPattern(` *`) + imapConn.Expect("From: ") + imapConn.Expect("To: ") + imapConn.Expect("Subject: Hi!") + imapConn.Expect("") + imapConn.Expect("Hi!") + imapConn.Expect(")") + imapConn.ExpectPattern(`. OK *`) +} + +func TestImapsqlDeliveryMap(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("imap") + t.Port("smtp") + t.Config(` + storage.imapsql test_store { + delivery_map email_localpart + auth_normalize precis + + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth dummy + storage &test_store + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + + deliver_to &test_store + } + `) + t.Run(2) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN testusr 1234") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) + + smtpConn := t.Conn("smtp") + defer smtpConn.Close() + smtpConn.SMTPNegotation("localhost", nil, nil) + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + smtpConn.Writeln("Hi!") + smtpConn.Writeln(".") + smtpConn.ExpectPattern("2*") + + time.Sleep(500 * time.Millisecond) + + imapConn.Writeln(". NOOP") + imapConn.ExpectPattern(`\* 1 EXISTS`) + imapConn.ExpectPattern(`\* 1 RECENT`) + imapConn.ExpectPattern(". OK *") +} + +func TestImapsqlAuthMap(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("imap") + t.Port("smtp") + t.Config(` + storage.imapsql test_store { + auth_map regexp "(.*)" "$1@maddy.test" + auth_normalize precis + + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth dummy + storage &test_store + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + + deliver_to &test_store + } + `) + t.Run(2) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN testusr 1234") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) + + smtpConn := t.Conn("smtp") + defer smtpConn.Close() + smtpConn.SMTPNegotation("localhost", nil, nil) + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + smtpConn.Writeln("Hi!") + smtpConn.Writeln(".") + smtpConn.ExpectPattern("2*") + + time.Sleep(500 * time.Millisecond) + + imapConn.Writeln(". NOOP") + imapConn.ExpectPattern(`\* 1 EXISTS`) + imapConn.ExpectPattern(`\* 1 RECENT`) + imapConn.ExpectPattern(". OK *") +} diff --git a/tests/issue327_test.go b/tests/issue327_test.go new file mode 100644 index 0000000..b766424 --- /dev/null +++ b/tests/issue327_test.go @@ -0,0 +1,91 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "strconv" + "testing" + "time" + + "github.com/foxcpp/maddy/internal/testutils" + "github.com/foxcpp/maddy/tests" +) + +func TestIssue327(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + tgtPort := t.Port("target") + t.Config(` + hostname mx.maddy.test + tls off + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + deliver_to queue outbound_queue { + target remote { } # it will fail + autogenerated_msg_domain maddy.test + bounce { + check { + spf + } + deliver_to lmtp tcp://127.0.0.1:{env:TEST_PORT_target} + } + } + }`) + t.Run(1) + defer t.Close() + + be, s := testutils.SMTPServer(tt, "127.0.0.1:"+strconv.Itoa(int(tgtPort))) + s.LMTP = true + be.LMTPDataErr = []error{nil, nil} + defer s.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("client.maddy.test", nil, nil) + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") + + for i := 0; i < 5; i++ { + time.Sleep(1 * time.Second) + if len(be.Messages) != 0 { + break + } + } + + if len(be.Messages) != 1 { + t.Fatal("No DSN sent?", len(be.Messages)) + } +} diff --git a/tests/limits_test.go b/tests/limits_test.go new file mode 100644 index 0000000..e219add --- /dev/null +++ b/tests/limits_test.go @@ -0,0 +1,107 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "testing" + + "github.com/foxcpp/maddy/tests" +) + +func TestConcurrencyLimit(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject no + limits { + all concurrency 1 + } + + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + c1 := t.Conn("smtp") + defer c1.Close() + c1.SMTPNegotation("localhost", nil, nil) + c1.Writeln("MAIL FROM:") + c1.ExpectPattern("250 *") + // Down on semaphore. + + c2 := t.Conn("smtp") + defer c2.Close() + c2.SMTPNegotation("localhost", nil, nil) + c1.Writeln("MAIL FROM:") + // Temporary error due to lock timeout. + c1.ExpectPattern("451 *") +} + +func TestPerIPConcurrency(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject no + limits { + ip concurrency 1 + } + + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + c1 := t.Conn("smtp") + defer c1.Close() + c1.SMTPNegotation("localhost", nil, nil) + c1.Writeln("MAIL FROM:") + c1.ExpectPattern("250 *") + // Down on semaphore. + + c3 := t.Conn4("127.0.0.2", "smtp") + defer c3.Close() + c3.SMTPNegotation("localhost", nil, nil) + c3.Writeln("MAIL FROM:") + c3.ExpectPattern("250 *") + // Down on semaphore (different IP). + + c2 := t.Conn("smtp") + defer c2.Close() + c2.SMTPNegotation("localhost", nil, nil) + c1.Writeln("MAIL FROM:") + // Temporary error due to lock timeout. + c1.ExpectPattern("451 *") +} diff --git a/tests/lmtp_test.go b/tests/lmtp_test.go new file mode 100644 index 0000000..8e29a93 --- /dev/null +++ b/tests/lmtp_test.go @@ -0,0 +1,181 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "strconv" + "strings" + "testing" + + "github.com/foxcpp/maddy/internal/testutils" + "github.com/foxcpp/maddy/tests" +) + +func TestLMTPServer_Is_Actually_LMTP(tt *testing.T) { + tt.Parallel() + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("lmtp") + t.Config(` + lmtp tcp://127.0.0.1:{env:TEST_PORT_lmtp} { + hostname mx.maddy.test + tls off + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + c := t.Conn("lmtp") + defer c.Close() + + c.Writeln("LHLO client.maddy.test") + c.ExpectPattern("220 *") +capsloop: + for { + line, err := c.Readln() + if err != nil { + t.Fatal("I/O error:", err) + } + switch { + case strings.HasPrefix(line, "250-"): + case strings.HasPrefix(line, "250 "): + break capsloop + default: + t.Fatal("Unexpected deply:", line) + } + } + + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") +} + +func TestLMTPClient_Is_Actually_LMTP(tt *testing.T) { + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + tgtPort := t.Port("target") + t.Config(` + hostname mx.maddy.test + tls off + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + deliver_to lmtp tcp://127.0.0.1:{env:TEST_PORT_target} + }`) + t.Run(1) + defer t.Close() + + be, s := testutils.SMTPServer(tt, "127.0.0.1:"+strconv.Itoa(int(tgtPort))) + s.LMTP = true + be.LMTPDataErr = []error{nil, nil} + defer s.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("client.maddy.test", nil, nil) + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") + + if be.SessionCounter != 1 { + t.Fatal("No actual connection made?", be.SessionCounter) + } +} + +func TestLMTPClient_Issue308(tt *testing.T) { + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + tgtPort := t.Port("target") + t.Config(` + hostname mx.maddy.test + tls off + + target.lmtp local_mailboxes { + targets tcp://127.0.0.1:{env:TEST_PORT_target} + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + deliver_to &local_mailboxes + }`) + t.Run(1) + defer t.Close() + + be, s := testutils.SMTPServer(tt, "127.0.0.1:"+strconv.Itoa(int(tgtPort))) + s.LMTP = true + be.LMTPDataErr = []error{nil, nil} + defer s.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("client.maddy.test", nil, nil) + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") + + if be.SessionCounter != 1 { + t.Fatal("No actual connection made?", be.SessionCounter) + } +} diff --git a/tests/mta_test.go b/tests/mta_test.go new file mode 100644 index 0000000..21ac001 --- /dev/null +++ b/tests/mta_test.go @@ -0,0 +1,135 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "net" + "strconv" + "testing" + + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/internal/testutils" + "github.com/foxcpp/maddy/tests" +) + +func TestMTA_Outbound(tt *testing.T) { + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + "example.invalid.": { + MX: []net.MX{{Host: "mx.example.invalid.", Pref: 10}}, + }, + "mx.example.invalid.": { + A: []string{"127.0.0.1"}, + }, + }) + t.Port("smtp") + tgtPort := t.Port("remote_smtp") + t.Config(` + hostname mx.maddy.test + tls off + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + deliver_to remote + }`) + t.Run(1) + defer t.Close() + + be, s := testutils.SMTPServer(tt, "127.0.0.1:"+strconv.Itoa(int(tgtPort))) + defer s.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("client.maddy.test", nil, nil) + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") + + if be.SessionCounter != 1 { + t.Fatal("No actual connection made?", be.SessionCounter) + } +} + +func TestIssue321(tt *testing.T) { + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + "example.invalid.": { + AD: true, + A: []string{"127.0.0.1"}, + }, + }) + t.Port("smtp") + tgtPort := t.Port("remote_smtp") + t.Config(` + hostname mx.maddy.test + tls off + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + deliver_to remote { + mx_auth { + dnssec + dane + } + } + }`) + t.Run(1) + defer t.Close() + + be, s := testutils.SMTPServer(tt, "127.0.0.1:"+strconv.Itoa(int(tgtPort))) + defer s.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("client.maddy.test", nil, nil) + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") + + if be.SessionCounter != 1 { + t.Fatal("No actual connection made?", be.SessionCounter) + } +} diff --git a/tests/multiple_domains_test.go b/tests/multiple_domains_test.go new file mode 100644 index 0000000..6b29c3f --- /dev/null +++ b/tests/multiple_domains_test.go @@ -0,0 +1,340 @@ +//go:build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2025 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "testing" + + "github.com/foxcpp/maddy/tests" +) + +// Test cases based on https://maddy.email/multiple-domains/ + +func TestMultipleDomains_SeparateNamespace(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("submission") + t.Port("imap") + t.Config(` + tls off + hostname test.maddy.email + + auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } + } + storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db + } + + submission tcp://0.0.0.0:{env:TEST_PORT_submission} { + auth &local_authdb + reject + } + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + auth &local_authdb + storage &local_mailboxes + } + `) + + t.MustRunCLIGroup( + []string{"creds", "create", "-p", "user1", "user1@test1.maddy.email"}, + []string{"creds", "create", "-p", "user2", "user2@test1.maddy.email"}, + []string{"creds", "create", "-p", "user3", "user1@test2.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user1@test1.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user2@test1.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user1@test2.maddy.email"}, + ) + t.Run(2) + + user1 := t.Conn("imap") + defer user1.Close() + user1.ExpectPattern(`\* OK *`) + user1.Writeln(`. LOGIN user1@test1.maddy.email user1`) + user1.ExpectPattern(`. OK *`) + user1.Writeln(`. CREATE user1`) + user1.ExpectPattern(`. OK *`) + + user1SMTP := t.Conn("submission") + defer user1SMTP.Close() + user1SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user1SMTP.SMTPPlainAuth("user1@test1.maddy.email", "user1", true) + + user2 := t.Conn("imap") + defer user2.Close() + user2.ExpectPattern(`\* OK *`) + user2.Writeln(`. LOGIN user2@test1.maddy.email user2`) + user2.ExpectPattern(`. OK *`) + user2.Writeln(`. CREATE user2`) + user2.ExpectPattern(`. OK *`) + + user2SMTP := t.Conn("submission") + defer user2SMTP.Close() + user2SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user2SMTP.SMTPPlainAuth("user2@test1.maddy.email", "user2", true) + + user3 := t.Conn("imap") + defer user3.Close() + user3.ExpectPattern(`\* OK *`) + user3.Writeln(`. LOGIN user1@test2.maddy.email user3`) + user3.ExpectPattern(`. OK *`) + user3.Writeln(`. CREATE user3`) + user3.ExpectPattern(`. OK *`) + + user3SMTP := t.Conn("submission") + defer user3SMTP.Close() + user3SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user3SMTP.SMTPPlainAuth("user1@test2.maddy.email", "user3", true) + + user1.Writeln(`. LIST "" "*"`) + user1.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user1.Expect(`* LIST (\HasNoChildren) "." "user1"`) + user1.ExpectPattern(". OK *") + + user2.Writeln(`. LIST "" "*"`) + user2.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user2.Expect(`* LIST (\HasNoChildren) "." "user2"`) + user2.ExpectPattern(". OK *") + + user3.Writeln(`. LIST "" "*"`) + user3.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user3.Expect(`* LIST (\HasNoChildren) "." "user3"`) + user3.ExpectPattern(". OK *") +} + +func TestMultipleDomains_SharedCredentials_DistinctMailboxes(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("submission") + t.Port("imap") + t.Config(` + tls off + hostname test.maddy.email + auth_map email_localpart + + auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } + } + storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db + } + + submission tcp://0.0.0.0:{env:TEST_PORT_submission} { + auth &local_authdb + reject + } + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + auth &local_authdb + storage &local_mailboxes + } + `) + + t.MustRunCLIGroup( + []string{"creds", "create", "-p", "user1", "user1"}, + []string{"creds", "create", "-p", "user2", "user2"}, + []string{"imap-acct", "create", "--no-specialuse", "user1@test1.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user2@test1.maddy.email"}, + []string{"imap-acct", "create", "--no-specialuse", "user1@test2.maddy.email"}, + ) + t.Run(2) + + user1 := t.Conn("imap") + defer user1.Close() + user1.ExpectPattern(`\* OK *`) + user1.Writeln(`. LOGIN user1@test1.maddy.email user1`) + user1.ExpectPattern(`. OK *`) + user1.Writeln(`. CREATE user1`) + user1.ExpectPattern(`. OK *`) + + user1SMTP := t.Conn("submission") + defer user1SMTP.Close() + user1SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user1SMTP.SMTPPlainAuth("user1@test1.maddy.email", "user1", true) + + user2 := t.Conn("imap") + defer user2.Close() + user2.ExpectPattern(`\* OK *`) + user2.Writeln(`. LOGIN user2@test1.maddy.email user2`) + user2.ExpectPattern(`. OK *`) + user2.Writeln(`. CREATE user2`) + user2.ExpectPattern(`. OK *`) + + user2SMTP := t.Conn("submission") + defer user2SMTP.Close() + user2SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user2SMTP.SMTPPlainAuth("user2@test1.maddy.email", "user2", true) + + user3 := t.Conn("imap") + defer user3.Close() + user3.ExpectPattern(`\* OK *`) + user3.Writeln(`. LOGIN user1@test2.maddy.email user1`) + user3.ExpectPattern(`. OK *`) + user3.Writeln(`. CREATE user3`) + user3.ExpectPattern(`. OK *`) + + user3SMTP := t.Conn("submission") + defer user3SMTP.Close() + user3SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user3SMTP.SMTPPlainAuth("user1@test2.maddy.email", "user1", true) + + user1.Writeln(`. LIST "" "*"`) + user1.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user1.Expect(`* LIST (\HasNoChildren) "." "user1"`) + user1.ExpectPattern(". OK *") + + user2.Writeln(`. LIST "" "*"`) + user2.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user2.Expect(`* LIST (\HasNoChildren) "." "user2"`) + user2.ExpectPattern(". OK *") + + user3.Writeln(`. LIST "" "*"`) + user3.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user3.Expect(`* LIST (\HasNoChildren) "." "user3"`) + user3.ExpectPattern(". OK *") +} + +func TestMultipleDomains_SharedCredentials_SharedMailboxes(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("submission") + t.Port("imap") + t.Config(` + tls off + hostname test.maddy.email + auth_map email_localpart_optional + + auth.pass_table local_authdb { + table sql_table { + driver sqlite3 + dsn credentials.db + table_name passwords + } + } + storage.imapsql local_mailboxes { + driver sqlite3 + dsn imapsql.db + + delivery_map email_localpart_optional + } + + submission tcp://0.0.0.0:{env:TEST_PORT_submission} { + auth &local_authdb + reject + } + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + auth &local_authdb + storage &local_mailboxes + + storage_map email_localpart_optional + } + `) + + t.MustRunCLIGroup( + []string{"creds", "create", "-p", "user1", "user1"}, + []string{"creds", "create", "-p", "user2", "user2"}, + []string{"imap-acct", "create", "--no-specialuse", "user1"}, + []string{"imap-acct", "create", "--no-specialuse", "user2"}, + ) + t.Run(2) + + user1 := t.Conn("imap") + defer user1.Close() + user1.ExpectPattern(`\* OK *`) + user1.Writeln(`. LOGIN user1 user1`) + user1.ExpectPattern(`. OK *`) + user1.Writeln(`. CREATE user1`) + user1.ExpectPattern(`. OK *`) + + user1SMTP := t.Conn("submission") + defer user1SMTP.Close() + user1SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user1SMTP.SMTPPlainAuth("user1", "user1", true) + + user2 := t.Conn("imap") + defer user2.Close() + user2.ExpectPattern(`\* OK *`) + user2.Writeln(`. LOGIN user2@test1.maddy.email user2`) + user2.ExpectPattern(`. OK *`) + user2.Writeln(`. CREATE user2`) + user2.ExpectPattern(`. OK *`) + + user2SMTP := t.Conn("submission") + defer user2SMTP.Close() + user2SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user2SMTP.SMTPPlainAuth("user2", "user2", true) + + user12 := t.Conn("imap") + defer user12.Close() + user12.ExpectPattern(`\* OK *`) + user12.Writeln(`. LOGIN user1@test2.maddy.email user1`) + user12.ExpectPattern(`. OK *`) + user12.Writeln(`. CREATE user12`) + user12.ExpectPattern(`. OK *`) + + user13 := t.Conn("imap") + defer user13.Close() + user13.ExpectPattern(`\* OK *`) + user13.Writeln(`. LOGIN user1@test.maddy.email user1`) + user13.ExpectPattern(`. OK *`) + user13.Writeln(`. CREATE user13`) + user13.ExpectPattern(`. OK *`) + + user12SMTP := t.Conn("submission") + defer user12SMTP.Close() + user12SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user12SMTP.SMTPPlainAuth("user1", "user1", true) + + user13SMTP := t.Conn("submission") + defer user13SMTP.Close() + user13SMTP.SMTPNegotation("localhost", []string{"AUTH PLAIN"}, nil) + user13SMTP.SMTPPlainAuth("user1@test.maddy.email", "user1", true) + + user1.Writeln(`. LIST "" "*"`) + user1.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user1.Expect(`* LIST (\HasNoChildren) "." "user1"`) + user1.Expect(`* LIST (\HasNoChildren) "." "user12"`) + user1.Expect(`* LIST (\HasNoChildren) "." "user13"`) + user1.ExpectPattern(". OK *") + + user2.Writeln(`. LIST "" "*"`) + user2.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user2.Expect(`* LIST (\HasNoChildren) "." "user2"`) + user2.ExpectPattern(". OK *") + + user12.Writeln(`. LIST "" "*"`) + user12.Expect(`* LIST (\HasNoChildren) "." INBOX`) + user12.Expect(`* LIST (\HasNoChildren) "." "user1"`) + user12.Expect(`* LIST (\HasNoChildren) "." "user12"`) + user12.Expect(`* LIST (\HasNoChildren) "." "user13"`) + user12.ExpectPattern(". OK *") +} diff --git a/tests/replace_addr_test.go b/tests/replace_addr_test.go new file mode 100644 index 0000000..c898071 --- /dev/null +++ b/tests/replace_addr_test.go @@ -0,0 +1,245 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "testing" + + "github.com/foxcpp/maddy/tests" +) + +func TestReplaceAddr_Rcpt(tt *testing.T) { + test := func(name, cfg string) { + tt.Run(name, func(tt *testing.T) { + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(cfg) + t.Run(1) + defer t.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("client.maddy.test", nil, nil) + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") + }) + } + + test("inline", ` + hostname mx.maddy.test + tls off + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + modify { + replace_rcpt static { + entry a@maddy.test b@maddy.test + } + } + destination a@maddy.test { + reject + } + destination b@maddy.test { + deliver_to dummy + } + default_destination { + reject + } + }`) + test("inline qualified", ` + hostname mx.maddy.test + tls off + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + modify { + modify.replace_rcpt static { + entry a@maddy.test b@maddy.test + } + } + destination a@maddy.test { + reject + } + destination b@maddy.test { + deliver_to dummy + } + default_destination { + reject + } + }`) + + // FIXME: Not implemented + // test("external", ` + // hostname mx.maddy.test + // tls off + + // modify.replace_rcpt local_aliases { + // table static { + // entry a@maddy.test b@maddy.test + // } + // } + + // smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + // modify { + // &local_aliases + // } + // source a@maddy.test { + // destination a@maddy.test { + // reject + // } + // destination b@maddy.test { + // deliver_to dummy + // } + // default_destination { + // reject + // } + // } + // default_source { + // reject + // } + // }`) +} + +func TestReplaceAddr_Sender(tt *testing.T) { + test := func(name, cfg string) { + tt.Run(name, func(tt *testing.T) { + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(cfg) + t.Run(1) + defer t.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("client.maddy.test", nil, nil) + c.Writeln("MAIL FROM:") + c.ExpectPattern("250 *") + c.Writeln("RCPT TO:") + c.ExpectPattern("250 *") + c.Writeln("DATA") + c.ExpectPattern("354 *") + c.Writeln("From: ") + c.Writeln("To: ") + c.Writeln("Subject: Hello!") + c.Writeln("") + c.Writeln("Hello!") + c.Writeln(".") + c.ExpectPattern("250 2.0.0 OK: queued") + c.Writeln("QUIT") + c.ExpectPattern("221 *") + }) + } + + test("inline", ` + hostname mx.maddy.test + tls off + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + modify { + replace_sender static { + entry a@maddy.test b@maddy.test + } + } + source a@maddy.test { + reject + } + source b@maddy.test { + destination a@maddy.test { + deliver_to dummy + } + default_destination { + reject + } + } + default_source { + reject + } + }`) + test("inline qualified", ` + hostname mx.maddy.test + tls off + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + modify { + modify.replace_sender static { + entry a@maddy.test b@maddy.test + } + } + source a@maddy.test { + reject + } + source b@maddy.test { + destination a@maddy.test { + deliver_to dummy + } + default_destination { + reject + } + } + default_source { + reject + } + }`) + // FIXME: Not implemented + // test("external", ` + // hostname mx.maddy.test + // tls off + + // modify.replace_sender local_aliases { + // table static { + // entry a@maddy.test b@maddy.test + // } + // } + + // smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + // modify { + // &local_aliases + // } + // source a@maddy.test { + // reject + // } + // source b@maddy.test { + // destination a@maddy.test { + // deliver_to dummy + // } + // default_destination { + // reject + // } + // } + // default_source { + // reject + // } + // }`) +} diff --git a/tests/run.sh b/tests/run.sh new file mode 100755 index 0000000..afbc802 --- /dev/null +++ b/tests/run.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +if [ -z "$GO" ]; then + export GO=go +fi + +./build_cover.sh + +clean() { + rm -f /tmp/maddy-coverage-report* +} +trap clean EXIT + +$GO test -tags integration -integration.executable ./maddy.cover -integration.coverprofile /tmp/maddy-coverage-report "$@" +$GO run gocovcat.go /tmp/maddy-coverage-report* > coverage.out diff --git a/tests/smtp_autobuffer_test.go b/tests/smtp_autobuffer_test.go new file mode 100644 index 0000000..58ef341 --- /dev/null +++ b/tests/smtp_autobuffer_test.go @@ -0,0 +1,347 @@ +//go:build integration && cgo && !nosqlite3 +// +build integration,cgo,!nosqlite3 + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "strings" + "testing" + "time" + + "github.com/foxcpp/maddy/tests" +) + +func TestSMTPEndpoint_LargeMessage(tt *testing.T) { + // Send 1.44 MiB message to verify it being handled correctly + // everywhere. + // (Issue 389) + + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("imap") + t.Port("smtp") + t.Config(` + storage.imapsql test_store { + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth dummy + storage &test_store + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + + deliver_to &test_store + } + `) + t.Run(2) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN testusr@maddy.test 1234") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) + + smtpConn := t.Conn("smtp") + defer smtpConn.Close() + smtpConn.SMTPNegotation("localhost", nil, nil) + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + for i := 0; i < 3000; i++ { + smtpConn.Writeln(strings.Repeat("A", 500)) + } + // 3000*502 ~ 1.44 MiB not including header side + smtpConn.Writeln(".") + + time.Sleep(500 * time.Millisecond) + + imapConn.Writeln(". NOOP") + imapConn.ExpectPattern(`\* 1 EXISTS`) + imapConn.ExpectPattern(`\* 1 RECENT`) + imapConn.ExpectPattern(". OK *") + + imapConn.Writeln(". FETCH 1 (BODY.PEEK[])") + imapConn.ExpectPattern(`\* 1 FETCH (BODY\[\] {1506312}*`) + imapConn.Expect(`Delivered-To: testusr@maddy.test`) + imapConn.Expect(`Return-Path: `) + imapConn.ExpectPattern(`Received: from localhost (client.maddy.test \[` + tests.DefaultSourceIP.String() + `\]) by maddy.test`) + imapConn.ExpectPattern(` (envelope-sender ) with ESMTP id *; *`) + imapConn.ExpectPattern(` *`) + imapConn.Expect("From: ") + imapConn.Expect("To: ") + imapConn.Expect("Subject: Hi!") + imapConn.Expect("") + for i := 0; i < 3000; i++ { + imapConn.Expect(strings.Repeat("A", 500)) + } + imapConn.Expect(")") + imapConn.ExpectPattern(`. OK *`) +} + +func TestSMTPEndpoint_FileBuffer(tt *testing.T) { + run := func(tt *testing.T, bufferOpt string) { + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("imap") + t.Port("smtp") + t.Config(` + storage.imapsql test_store { + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth dummy + storage &test_store + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + buffer ` + bufferOpt + ` + + deliver_to &test_store + } + `) + t.Run(2) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN testusr@maddy.test 1234") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) + + smtpConn := t.Conn("smtp") + defer smtpConn.Close() + smtpConn.SMTPNegotation("localhost", nil, nil) + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + smtpConn.Writeln("AAAAABBBBBB") + smtpConn.Writeln(".") + smtpConn.ExpectPattern("2*") + + time.Sleep(500 * time.Millisecond) + + imapConn.Writeln(". NOOP") + imapConn.ExpectPattern(`\* 1 EXISTS`) + imapConn.ExpectPattern(`\* 1 RECENT`) + imapConn.ExpectPattern(". OK *") + + imapConn.Writeln(". FETCH 1 (BODY.PEEK[])") + imapConn.ExpectPattern(`\* 1 FETCH (BODY\[\] {*}*`) + imapConn.Expect(`Delivered-To: testusr@maddy.test`) + imapConn.Expect(`Return-Path: `) + imapConn.ExpectPattern(`Received: from localhost (client.maddy.test \[` + tests.DefaultSourceIP.String() + `\]) by maddy.test`) + imapConn.ExpectPattern(` (envelope-sender ) with ESMTP id *; *`) + imapConn.ExpectPattern(` *`) + imapConn.Expect("From: ") + imapConn.Expect("To: ") + imapConn.Expect("Subject: Hi!") + imapConn.Expect("") + imapConn.Expect("AAAAABBBBBB") + imapConn.Expect(")") + imapConn.ExpectPattern(`. OK *`) + } + + tt.Run("ram", func(tt *testing.T) { run(tt, "ram") }) + tt.Run("fs", func(tt *testing.T) { run(tt, "fs") }) +} + +func TestSMTPEndpoint_Autobuffer(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + + t.DNS(nil) + t.Port("imap") + t.Port("smtp") + t.Config(` + storage.imapsql test_store { + driver sqlite3 + dsn imapsql.db + } + + imap tcp://127.0.0.1:{env:TEST_PORT_imap} { + tls off + + auth dummy + storage &test_store + } + + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname maddy.test + tls off + buffer auto 5b + + deliver_to &test_store + } + `) + t.Run(2) + defer t.Close() + + imapConn := t.Conn("imap") + defer imapConn.Close() + imapConn.ExpectPattern(`\* OK *`) + imapConn.Writeln(". LOGIN testusr@maddy.test 1234") + imapConn.ExpectPattern(". OK *") + imapConn.Writeln(". SELECT INBOX") + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`\* *`) + imapConn.ExpectPattern(`. OK *`) + + smtpConn := t.Conn("smtp") + defer smtpConn.Close() + smtpConn.SMTPNegotation("localhost", nil, nil) + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + smtpConn.Writeln("AAAAABBBBBB") + smtpConn.Writeln(".") + smtpConn.ExpectPattern("2*") + + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + smtpConn.Writeln(".") + smtpConn.ExpectPattern("2*") + + smtpConn.Writeln("MAIL FROM:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("RCPT TO:") + smtpConn.ExpectPattern("2*") + smtpConn.Writeln("DATA") + smtpConn.ExpectPattern("354 *") + smtpConn.Writeln("From: ") + smtpConn.Writeln("To: ") + smtpConn.Writeln("Subject: Hi!") + smtpConn.Writeln("") + smtpConn.Writeln("AAA") + smtpConn.Writeln(".") + smtpConn.ExpectPattern("2*") + + time.Sleep(500 * time.Millisecond) + + imapConn.Writeln(". NOOP") + // This will break with go-imap v2 upgrade merging updates. + imapConn.ExpectPattern(`\* 3 EXISTS`) + imapConn.ExpectPattern(`\* 3 RECENT`) + imapConn.ExpectPattern(". OK *") + + imapConn.Writeln(". FETCH 1:3 (BODY.PEEK[])") + imapConn.ExpectPattern(`\* 1 FETCH (BODY\[\] {*}*`) + imapConn.Expect(`Delivered-To: testusr@maddy.test`) + imapConn.Expect(`Return-Path: `) + imapConn.ExpectPattern(`Received: from localhost (client.maddy.test \[` + tests.DefaultSourceIP.String() + `\]) by maddy.test`) + imapConn.ExpectPattern(` (envelope-sender ) with ESMTP id *; *`) + imapConn.ExpectPattern(` *`) + imapConn.Expect("From: ") + imapConn.Expect("To: ") + imapConn.Expect("Subject: Hi!") + imapConn.Expect("") + imapConn.Expect("AAAAABBBBBB") + imapConn.Expect(")") + imapConn.ExpectPattern(`\* 2 FETCH (BODY\[\] {*}*`) + imapConn.Expect(`Delivered-To: testusr@maddy.test`) + imapConn.Expect(`Return-Path: `) + imapConn.ExpectPattern(`Received: from localhost (client.maddy.test \[` + tests.DefaultSourceIP.String() + `\]) by maddy.test`) + imapConn.ExpectPattern(` (envelope-sender ) with ESMTP id *; *`) + imapConn.ExpectPattern(` *`) + imapConn.Expect("From: ") + imapConn.Expect("To: ") + imapConn.Expect("Subject: Hi!") + imapConn.Expect("") + imapConn.Expect(")") + imapConn.ExpectPattern(`\* 3 FETCH (BODY\[\] {*}*`) + imapConn.Expect(`Delivered-To: testusr@maddy.test`) + imapConn.Expect(`Return-Path: `) + imapConn.ExpectPattern(`Received: from localhost (client.maddy.test \[` + tests.DefaultSourceIP.String() + `\]) by maddy.test`) + imapConn.ExpectPattern(` (envelope-sender ) with ESMTP id *; *`) + imapConn.ExpectPattern(` *`) + imapConn.Expect("From: ") + imapConn.Expect("To: ") + imapConn.Expect("Subject: Hi!") + imapConn.Expect("") + imapConn.Expect("AAA") + imapConn.Expect(")") + imapConn.ExpectPattern(`. OK *`) +} diff --git a/tests/smtp_test.go b/tests/smtp_test.go new file mode 100644 index 0000000..d897227 --- /dev/null +++ b/tests/smtp_test.go @@ -0,0 +1,667 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "errors" + "fmt" + "io/ioutil" + "path/filepath" + "strings" + "testing" + + "github.com/foxcpp/go-mockdns" + "github.com/foxcpp/maddy/tests" +) + +func TestCheckRequireTLS(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls self_signed + + defer_sender_reject no + + check { + require_tls + } + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.SMTPNegotation("localhost", nil, nil) + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("550 5.7.1 *") + conn.Writeln("STARTTLS") + conn.ExpectPattern("220 *") + conn.TLS() + conn.SMTPNegotation("localhost", nil, nil) + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("250 *") + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestProxyProtocolTrustedSource(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + "one.maddy.test.": { + TXT: []string{"v=spf1 ip4:127.0.0.17 -all"}, + }, + }) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + proxy_protocol { + trust ` + tests.DefaultSourceIP.String() + ` ::1/128 + tls off + } + + defer_sender_reject no + + check { + spf { + enforce_early yes + fail_action reject + } + } + + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.Writeln(fmt.Sprintf("PROXY TCP4 127.0.0.17 %s 12345 %d", tests.DefaultSourceIP.String(), t.Port("smtp"))) + conn.SMTPNegotation("localhost", nil, nil) + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("250 *") + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestProxyProtocolUntrustedSource(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + "one.maddy.test.": { + TXT: []string{"v=spf1 ip4:127.0.0.17 -all"}, + }, + }) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + proxy_protocol { + trust fe80::bad/128 + tls off + } + + defer_sender_reject no + + check { + spf { + enforce_early yes + fail_action reject + } + } + + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.Writeln(fmt.Sprintf("PROXY TCP4 127.0.0.17 %s 12345 %d", tests.DefaultSourceIP.String(), t.Port("smtp"))) + conn.SMTPNegotation("localhost", nil, nil) + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("550 *") + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestCheckSPF(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + "none.maddy.test.": { + TXT: []string{}, + }, + "pass.maddy.test.": { + TXT: []string{"v=spf1 +all"}, + }, + "neutral.maddy.test.": { + TXT: []string{"v=spf1 ?all"}, + }, + "fail.maddy.test.": { + TXT: []string{"v=spf1 -all"}, + }, + "softfail.maddy.test.": { + TXT: []string{"v=spf1 ~all"}, + }, + "permerr.maddy.test.": { + TXT: []string{"v=spf1 something_clever"}, + }, + "temperr.maddy.test.": { + Err: errors.New("IANA forgot to resign the root zone"), + }, + }) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject no + + check { + spf { + enforce_early yes + + none_action reject 551 + neutral_action reject + fail_action reject 552 + softfail_action reject 553 + permerr_action reject 554 + temperr_action reject 455 + } + } + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.SMTPNegotation("fail.maddy.test", nil, nil) + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("250 *") + conn.Writeln("RSET") + conn.ExpectPattern("250 *") + + // Actually checks fail.maddy.test. + conn.Writeln("MAIL FROM:<>") + conn.ExpectPattern("552 5.7.0 *") + + conn.SMTPNegotation("pass.maddy.test", nil, nil) + + conn.Writeln("MAIL FROM:<>") + conn.ExpectPattern("250 *") + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("551 5.7.0 *") + + // Also check the default enhanced code is meaningful. + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("550 5.7.23 *") + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("552 5.7.0 *") + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("553 5.7.0 *") + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("554 5.7.0 *") + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("455 4.7.0 *") + + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestSPF_DMARCDefer(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + "subdomain.maddy-dmarc.test.": { + TXT: []string{"v=spf1 -all"}, + }, + "maddy-dmarc.test.": { + TXT: []string{"v=spf1 -all"}, + }, + "_dmarc.maddy-dmarc.test.": { + TXT: []string{"v=DMARC1; p=reject; sp=none"}, + }, + "subdomain.maddy-dmarc2.test.": { + TXT: []string{"v=spf1 -all"}, + }, + "maddy-dmarc2.test.": { + TXT: []string{"v=spf1 -all"}, + }, + "_dmarc.maddy-dmarc2.test.": { + TXT: []string{"v=DMARC1; p=reject"}, + }, + "maddy-no-dmarc.test.": { + TXT: []string{"v=spf1 -all"}, + }, + "maddy-dmarc-lookup-fail.test.": { + TXT: []string{"v=spf1 -all"}, + }, + "_dmarc.maddy-dmarc-lookup-fail.test.": { + Err: errors.New("nop"), + }, + }) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject no + + check { + spf { + enforce_early no + + none_action ignore + neutral_action reject + fail_action reject + softfail_action reject + permerr_action reject + temperr_action reject + } + } + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.SMTPNegotation("localhost", nil, nil) + + msg := func(fromEnv, fromHdr, bodyRespPattern string) { + tt.Helper() + + conn.Writeln("MAIL FROM:<" + fromEnv + ">") + conn.ExpectPattern("250 *") + conn.Writeln("RCPT TO:") + conn.ExpectPattern("250 *") + conn.Writeln("DATA") + conn.ExpectPattern("354 *") + conn.Writeln("From: <" + fromHdr + ">") + conn.Writeln("") + conn.Writeln("Heya!") + conn.Writeln(".") + conn.ExpectPattern(bodyRespPattern) + } + + msg("test@subdomain.maddy-dmarc.test", "test@subdomain.maddy-dmarc.test", "550 *") + + // Malformed From domain, DMARC cannot work so use only SPF. + msg("test@subdomain.maddy-dmarc.test", "", "550 *") + + msg("test@subdomain.maddy-dmarc.test", "maddy-dmarc-lookup-fail.test", "550 *") + + // No actual DMARC check is done but SPF check results are not applied. + msg("test@maddy-dmarc.test", "test@maddy-dmarc.test", "250 *") + msg("test@maddy-dmarc2.test", "test@maddy-dmarc2.test", "250 *") + + msg("test@maddy-no-dmarc.test", "test@maddy-no-dmarc.test", "550 *") + + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestDNSBLConfig(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + tests.DefaultSourceIPRev + ".dnsbl.test.": { + A: []string{"127.0.0.127"}, + }, + "sender.test.dnsbl.test.": { + A: []string{"127.0.0.127"}, + }, + }) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject no + + check { + dnsbl { + reject_threshold 1 + + dnsbl.test { + client_ipv4 + mailfrom + } + } + } + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.SMTPNegotation("localhost", nil, nil) + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("554 5.7.0 Client identity is listed in the used DNSBL *") + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("554 5.7.0 Client identity is listed in the used DNSBL *") + + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestDNSBLConfig2(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(map[string]mockdns.Zone{ + tests.DefaultSourceIPRev + ".dnsbl2.test.": { + A: []string{"127.0.0.127"}, + }, + "sender.test.dnsbl.test.": { + A: []string{"127.0.0.127"}, + }, + }) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject no + + check { + dnsbl { + reject_threshold 1 + + dnsbl.test { + mailfrom + } + dnsbl2.test { + client_ipv4 + score -1 + } + } + } + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.SMTPNegotation("localhost", nil, nil) + + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("250 *") + + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestCheckAuthorizeSender(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + auth dummy + defer_sender_reject off + + source example1.org { + check { + authorize_sender { + auth_normalize precis_casefold + user_to_email static { + entry "test-user1" "test@example1.org" + entry "test-user2" "é@example1.org" + } + } + } + deliver_to dummy + } + source example2.org { + check { + authorize_sender { + auth_normalize precis_casefold + prepare_email static { + entry "alias-to-test@example2.org" "test@example2.org" + } + user_to_email static { + entry "test-user1" "test@example2.org" + entry "test-user2" "test2@example2.org" + } + } + } + deliver_to dummy + } + + default_source { + reject + } + }`) + t.Run(1) + defer t.Close() + + c := t.Conn("smtp") + c.SMTPNegotation("client.maddy.test", nil, nil) + c.SMTPPlainAuth("test-user2", "1", true) + c.Writeln("MAIL FROM:") + c.ExpectPattern("5*") // rejected - user is not test-user1 + c.Writeln("MAIL FROM:") + c.ExpectPattern("5*") // rejected - unknown email + c.Writeln("MAIL FROM: SMTPUTF8") + c.ExpectPattern("2*") // OK - é@example1.org belongs to test-user2 + c.Close() + + c = t.Conn("smtp") + c.SMTPNegotation("client.maddy.test", nil, nil) + c.SMTPPlainAuth("test-user1", "1", true) + c.Writeln("MAIL FROM:") + c.ExpectPattern("5*") // rejected - user is not test-user2 + c.Writeln("MAIL FROM:") + c.ExpectPattern("2*") // OK - test@example2.org belongs to test-user + c.Writeln("MAIL FROM:") + c.ExpectPattern("2*") // OK - test@example2.org belongs to test-user + c.Close() +} + +func TestCheckCommand(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + check { + command {env:TEST_PWD}/testdata/check_command.sh {sender} { + code 12 reject + } + } + deliver_to dummy + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.SMTPNegotation("localhost", nil, nil) + + // Note: Internally, messages are handled using LF line endings, being + // converted CRLF only when transfered over Internet protocols. + expectedMsg := "From: \n" + + "To: \n" + + "Subject: Hi there!\n" + + "\n" + + "Nice to meet you!\n" + submitMsg := func(conn *tests.Conn, from string) { + // Fairly trivial SMTP transaction. + conn.Writeln("MAIL FROM:<" + from + ">") + conn.ExpectPattern("250 *") + conn.Writeln("RCPT TO:") + conn.ExpectPattern("250 *") + conn.Writeln("DATA") + conn.ExpectPattern("354 *") + conn.Writeln("From: ") + conn.Writeln("To: ") + conn.Writeln("Subject: Hi there!") + conn.Writeln("") + conn.Writeln("Nice to meet you!") + conn.Writeln(".") + } + + t.Subtest("Message dump", func(t *tests.T) { + conn := conn.Rebind(t) + + submitMsg(conn, "testing@maddy.test") + conn.ExpectPattern("250 *") + + msgPath := filepath.Join(t.StateDir(), "msg") + msgContents, err := ioutil.ReadFile(msgPath) + if err != nil { + t.Fatal(err) + } + + if string(msgContents) != expectedMsg { + t.Log("Wrong message contents received by check script!") + t.Log("Actual:") + t.Log(msgContents) + t.Log("Expected:") + t.Log(expectedMsg) + } + }) + t.Subtest("Message dump + Add header", func(t *tests.T) { + conn := conn.Rebind(t) + + submitMsg(conn, "testing+addHeader@maddy.test") + conn.ExpectPattern("250 *") + + msgPath := filepath.Join(t.StateDir(), "msg") + msgContents, err := ioutil.ReadFile(msgPath) + if err != nil { + t.Fatal(err) + } + + expectedMsg := "X-Added-Header: 1\n" + expectedMsg + if string(msgContents) != expectedMsg { + t.Log("Wrong message contents received by check script!") + t.Log("Actual:") + t.Log(msgContents) + t.Log("Expected:") + t.Log(expectedMsg) + } + }) + t.Subtest("Body reject", func(t *tests.T) { + conn := conn.Rebind(t) + + submitMsg(conn, "testing+reject@maddy.test") + conn.ExpectPattern("550 *") + + msgPath := filepath.Join(t.StateDir(), "msg") + msgContents, err := ioutil.ReadFile(msgPath) + if err != nil { + t.Fatal(err) + } + + if string(msgContents) != expectedMsg { + t.Log("Wrong message contents received by check script!") + t.Log("Actual:") + t.Log(msgContents) + t.Log("Expected:") + t.Log([]byte(expectedMsg)) + } + }) + + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} + +func TestHeaderSizeConstraint(tt *testing.T) { + tt.Parallel() + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + deliver_to dummy + max_header_size 1K + } + `) + t.Run(1) + defer t.Close() + + conn := t.Conn("smtp") + defer conn.Close() + conn.SMTPNegotation("localhost", nil, nil) + conn.Writeln("MAIL FROM:") + conn.ExpectPattern("250 *") + conn.Writeln("RCPT TO:") + conn.ExpectPattern("250 *") + conn.Writeln("DATA") + conn.ExpectPattern("354 *") + conn.Writeln("From: ") + conn.Writeln("To: ") + conn.Writeln("Subject: " + strings.Repeat("A", 2*1024)) + conn.Writeln("") + conn.Writeln("Hi") + conn.Writeln(".") + + conn.ExpectPattern("552 5.3.4 Message header size exceeds limit *") + + conn.Writeln("QUIT") + conn.ExpectPattern("221 *") +} diff --git a/tests/stress_test.go b/tests/stress_test.go new file mode 100644 index 0000000..74ed268 --- /dev/null +++ b/tests/stress_test.go @@ -0,0 +1,411 @@ +//go:build integration +// +build integration + +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tests_test + +import ( + "sync" + "testing" + "time" + + "github.com/foxcpp/maddy/tests" +) + +func floodSmtp(c *tests.Conn, commands, expectedPatterns []string, iterations int) { + for i := 0; i < iterations; i++ { + for i, cmd := range commands { + c.Writeln(cmd) + if expectedPatterns[i] != "" { + c.ExpectPattern(expectedPatterns[i]) + } + } + } +} + +func TestSMTPFlood_FullMsg_NoLimits_1Conn(tt *testing.T) { + tt.Parallel() + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "DATA", + "From: ", + "", + "Heya!", + ".", + }, []string{ + "250 *", + "250 *", + "354 *", + "", + "", + "", + "250 *", + }, 100) +} + +func TestSMTPFlood_FullMsg_NoLimits_10Conns(tt *testing.T) { + tt.Parallel() + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + wg := sync.WaitGroup{} + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "DATA", + "From: ", + "", + "Heya!", + ".", + }, []string{ + "250 *", + "250 *", + "354 *", + "", + "", + "", + "250 *", + }, 100) + t.Log("Done") + }() + } + + wg.Wait() +} + +func TestSMTPFlood_EnvelopeAbort_NoLimits_10Conns(tt *testing.T) { + tt.Parallel() + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + wg := sync.WaitGroup{} + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "RSET", + }, []string{ + "250 *", + "250 *", + "250 *", + }, 100) + t.Log("Done") + }() + } + + wg.Wait() +} + +func TestSMTPFlood_EnvelopeAbort_Ratelimited(tt *testing.T) { + tt.Parallel() + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + limits { + all rate 10 1s + } + + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + conns := 5 + msgsPerConn := 10 + expectedRate := 10 + slip := 10 + + start := time.Now() + + wg := sync.WaitGroup{} + for i := 0; i < conns; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "RSET", + }, []string{ + "250 *", + "250 *", + "250 *", + }, msgsPerConn) + t.Log("Done") + }() + } + + wg.Wait() + end := time.Now() + + t.Log("Sent", conns*msgsPerConn, "messages using", conns, "connections") + t.Log("Took", end.Sub(start)) + + effectiveRate := float64(conns*msgsPerConn) / end.Sub(start).Seconds() + if effectiveRate > float64(expectedRate+slip) { + t.Fatal("Effective rate is significantly bigger than limit:", effectiveRate) + } + t.Log("Effective rate:", effectiveRate) +} + +func TestSMTPFlood_FullMsg_Ratelimited_PerSource(tt *testing.T) { + tt.Parallel() + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject false + + limits { + source rate 10 1s + } + + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + conns := 5 + msgsPerConn := 10 + expectedRate := 10 + slip := 10 + + start := time.Now() + + wg := sync.WaitGroup{} + for i := 0; i < conns; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "DATA", + "From: ", + "", + "Heya!", + ".", + }, []string{ + "250 *", + "250 *", + "354 *", + "", + "", + "", + "250 *", + }, msgsPerConn) + t.Log("Done") + }() + } + for i := 0; i < conns; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := t.Conn("smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "DATA", + "From: ", + "", + "Heya!", + ".", + }, []string{ + "250 *", + "250 *", + "354 *", + "", + "", + "", + "250 *", + }, msgsPerConn) + t.Log("Done") + }() + } + + wg.Wait() + end := time.Now() + + t.Log("Sent", conns*msgsPerConn, "messages using", conns, "connections") + t.Log("Took", end.Sub(start)) + + effectiveRate := float64(conns*msgsPerConn*2) / end.Sub(start).Seconds() + // Expect the rate twice since we send from two sources. + if effectiveRate > float64(expectedRate*2+slip) { + t.Fatal("Effective rate is significantly bigger than limit:", effectiveRate) + } + t.Log("Effective rate:", effectiveRate) +} + +func TestSMTPFlood_EnvelopeAbort_Ratelimited_PerIP(tt *testing.T) { + tt.Parallel() + + t := tests.NewT(tt) + t.DNS(nil) + t.Port("smtp") + t.Config(` + smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} { + hostname mx.maddy.test + tls off + + defer_sender_reject false + + limits { + ip rate 10 1s + } + + deliver_to dummy + }`) + t.Run(1) + defer t.Close() + + conns := 2 + msgsPerConn := 50 + expectedRate := 10 + slip := 10 + + start := time.Now() + + wg := sync.WaitGroup{} + for i := 0; i < conns; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := t.Conn4("127.0.0.1", "smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "RSET", + }, []string{ + "250 *", + "250 *", + "250 *", + }, msgsPerConn) + t.Log("Done") + }() + } + for i := 0; i < conns; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c := t.Conn4("127.0.0.2", "smtp") + defer c.Close() + c.SMTPNegotation("helo.maddy.test", nil, nil) + floodSmtp(&c, []string{ + "MAIL FROM:", + "RCPT TO:", + "RSET", + }, []string{ + "250 *", + "250 *", + "250 *", + }, msgsPerConn) + t.Log("Done") + }() + } + + wg.Wait() + end := time.Now() + + t.Log("Sent", 2*conns*msgsPerConn, "messages using", conns*2, "connections") + t.Log("Took", end.Sub(start)) + + effectiveRate := float64(conns*msgsPerConn*2) / end.Sub(start).Seconds() + // Expect the rate twice since we send from two sources. + if effectiveRate > float64(expectedRate*2+slip) { + t.Fatal("Effective rate is significantly bigger than limit:", effectiveRate) + } + t.Log("Expected rate:", expectedRate*2) + t.Log("Effective rate:", effectiveRate) +} diff --git a/tests/t.go b/tests/t.go new file mode 100644 index 0000000..c1c04bc --- /dev/null +++ b/tests/t.go @@ -0,0 +1,484 @@ +/* +Maddy Mail Server - Composable all-in-one email server. +Copyright © 2019-2020 Max Mazurov , Maddy Mail Server contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Package tests provides the framework for integration testing of maddy. +// +// The packages core object is tests.T object that encapsulates all test +// state. It runs the server using test-provided configuration file and acts as +// a proxy for all interactions with the server. +package tests + +import ( + "bufio" + "bytes" + "flag" + "fmt" + "math/rand" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/foxcpp/go-mockdns" +) + +var ( + TestBinary = "./maddy" + CoverageOut string + DebugLog bool +) + +type T struct { + *testing.T + + testDir string + cfg string + + dnsServ *mockdns.Server + env []string + ports map[string]uint16 + portsRev map[uint16]string + + servProc *exec.Cmd +} + +func NewT(t *testing.T) *T { + return &T{ + T: t, + ports: map[string]uint16{}, + portsRev: map[uint16]string{}, + } +} + +// Config sets the configuration to use for the server. It must be called +// before Run. +func (t *T) Config(cfg string) { + t.Helper() + + if t.servProc != nil { + panic("tests: Config called after Run") + } + + t.cfg = cfg +} + +// DNS sets the DNS zones to emulate for the tested server instance. +// +// If it is not called before Run, DNS(nil) call is assumed which makes the +// mockdns server respond with NXDOMAIN to all queries. +func (t *T) DNS(zones map[string]mockdns.Zone) { + t.Helper() + + if zones == nil { + zones = map[string]mockdns.Zone{} + } + if _, ok := zones["100.97.109.127.in-addr.arpa."]; !ok { + zones["100.97.109.127.in-addr.arpa."] = mockdns.Zone{PTR: []string{"client.maddy.test."}} + } + + if t.dnsServ != nil { + t.Log("NOTE: Multiple DNS calls, replacing the server instance...") + t.dnsServ.Close() + } + + dnsServ, err := mockdns.NewServer(zones, false) + if err != nil { + t.Fatal("Test configuration failed:", err) + } + dnsServ.Log = t + t.dnsServ = dnsServ +} + +// Port allocates the random TCP port for use by test. It will made accessible +// in the configuration via environment variables with name in the form +// TEST_PORT_name. +// +// If there is a port with name remote_smtp, it will be passed as the value for +// the -debug.smtpport parameter. +func (t *T) Port(name string) uint16 { + if port := t.ports[name]; port != 0 { + return port + } + + // TODO: Try to bind on port to test its usability. + port := rand.Int31n(45536) + 20000 + t.ports[name] = uint16(port) + t.portsRev[uint16(port)] = name + return uint16(port) +} + +func (t *T) Env(kv string) { + t.env = append(t.env, kv) +} + +func (t *T) ensureCanRun() { + if t.cfg == "" { + panic("tests: Run called without configuration set") + } + if t.dnsServ == nil { + // If there is no DNS zones set in test - start a server that will + // respond with NXDOMAIN to all queries to avoid accidentally leaking + // any DNS queries to the real world. + t.Log("NOTE: Explicit DNS(nil) is recommended.") + t.DNS(nil) + + t.Cleanup(func() { + // Shutdown the DNS server after maddy to make sure it will not spend time + // timing out queries. + if err := t.dnsServ.Close(); err != nil { + t.Log("Unable to stop the DNS server:", err) + } + t.dnsServ = nil + }) + } + + // Setup file system, create statedir, runtimedir, write out config. + if t.testDir == "" { + testDir, err := os.MkdirTemp("", "maddy-tests-") + if err != nil { + t.Fatal("Test configuration failed:", err) + } + t.testDir = testDir + t.Log("using", t.testDir) + + if err := os.MkdirAll(filepath.Join(t.testDir, "statedir"), os.ModePerm); err != nil { + t.Fatal("Test configuration failed:", err) + } + if err := os.MkdirAll(filepath.Join(t.testDir, "runtimedir"), os.ModePerm); err != nil { + t.Fatal("Test configuration failed:", err) + } + + t.Cleanup(func() { + if !t.Failed() { + return + } + + t.Log("removing", t.testDir) + os.RemoveAll(t.testDir) + t.testDir = "" + }) + } + + configPreable := "state_dir " + filepath.Join(t.testDir, "statedir") + "\n" + + "runtime_dir " + filepath.Join(t.testDir, "runtimedir") + "\n\n" + + err := os.WriteFile(filepath.Join(t.testDir, "maddy.conf"), []byte(configPreable+t.cfg), os.ModePerm) + if err != nil { + t.Fatal("Test configuration failed:", err) + } +} + +func (t *T) buildCmd(additionalArgs ...string) *exec.Cmd { + // Assigning 0 by default will make outbound SMTP unusable. + remoteSmtp := "0" + if port := t.ports["remote_smtp"]; port != 0 { + remoteSmtp = strconv.Itoa(int(port)) + } + + args := []string{"-config", filepath.Join(t.testDir, "maddy.conf"), + "-debug.smtpport", remoteSmtp, + "-debug.dnsoverride", t.dnsServ.LocalAddr().String(), + "-log", "/tmp/test.log"} + + if CoverageOut != "" { + args = append(args, "-test.coverprofile", CoverageOut+"."+strconv.FormatInt(time.Now().UnixNano(), 16)) + } + if DebugLog { + args = append(args, "-debug") + } + + args = append(args, additionalArgs...) + + cmd := exec.Command(TestBinary, args...) + + pwd, err := os.Getwd() + if err != nil { + t.Fatal("Test configuration failed:", err) + } + + // Set environment variables. + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, + "TEST_PWD="+pwd, + "TEST_STATE_DIR="+filepath.Join(t.testDir, "statedir"), + "TEST_RUNTIME_DIR="+filepath.Join(t.testDir, "runtimedir"), + ) + for name, port := range t.ports { + cmd.Env = append(cmd.Env, fmt.Sprintf("TEST_PORT_%s=%d", name, port)) + } + cmd.Env = append(cmd.Env, t.env...) + + return cmd +} + +func (t *T) MustRunCLIGroup(args ...[]string) { + t.ensureCanRun() + + wg := sync.WaitGroup{} + for _, arg := range args { + wg.Add(1) + go func() { + defer wg.Done() + + _, err := t.RunCLI(arg...) + if err != nil { + t.Printf("maddy %v: %v", arg, err) + t.Fail() + } + }() + } + wg.Wait() +} + +func (t *T) MustRunCLI(args ...string) string { + s, err := t.RunCLI(args...) + if err != nil { + t.Fatalf("maddy %v: %v", args, err) + } + return s +} + +func (t *T) RunCLI(args ...string) (string, error) { + t.ensureCanRun() + cmd := t.buildCmd(args...) + + var stderr, stdout bytes.Buffer + cmd.Stderr = &stderr + cmd.Stdout = &stdout + + t.Log("launching maddy", cmd.Args) + if err := cmd.Run(); err != nil { + t.Log("Stderr:", stderr.String()) + t.Fatal("Test configuration failed:", err) + } + + t.Log("Stderr:", stderr.String()) + + return stdout.String(), nil +} + +// Run completes the configuration of test environment and starts the test server. +// +// T.Close should be called by the end of test to release any resources and +// shutdown the server. +// +// The parameter waitListeners specifies the amount of listeners the server is +// supposed to configure. Run() will block before all of them are up. +func (t *T) Run(waitListeners int) { + t.ensureCanRun() + cmd := t.buildCmd("run") + + // Capture maddy log and redirect it. + logOut, err := cmd.StderrPipe() + if err != nil { + t.Fatal("Test configuration failed:", err) + } + + t.Log("launching maddy", cmd.Args) + if err := cmd.Start(); err != nil { + t.Fatal("Test configuration failed:", err) + } + + // Log scanning goroutine checks for the "listening" messages and sends 'true' + // on the channel each time. + listeningMsg := make(chan bool) + + go func() { + defer logOut.Close() + defer close(listeningMsg) + scnr := bufio.NewScanner(logOut) + for scnr.Scan() { + line := scnr.Text() + + if strings.Contains(line, "listening on") { + listeningMsg <- true + line += " (test runner>listener wait trigger<)" + } + + t.Log("maddy:", line) + } + if err := scnr.Err(); err != nil { + t.Log("stderr I/O error:", err) + } + }() + + for i := 0; i < waitListeners; i++ { + if !<-listeningMsg { + t.Fatal("Log ended before all expected listeners are up. Start-up error?") + } + } + + t.servProc = cmd + + t.Cleanup(t.killServer) +} + +func (t *T) StateDir() string { + return filepath.Join(t.testDir, "statedir") +} + +func (t *T) RuntimeDir() string { + return filepath.Join(t.testDir, "statedir") +} + +func (t *T) killServer() { + if err := t.servProc.Process.Signal(os.Interrupt); err != nil { + t.Log("Unable to kill the server process:", err) + os.RemoveAll(t.testDir) + return // Return, as now it is pointless to wait for it. + } + + go func() { + time.Sleep(5 * time.Second) + if t.servProc != nil { + t.Log("Killing possibly hung server process") + t.servProc.Process.Kill() //nolint:errcheck + } + }() + + if err := t.servProc.Wait(); err != nil { + t.Error("The server did not stop cleanly, deadlock?") + } + + t.servProc = nil + + if err := os.RemoveAll(t.testDir); err != nil { + t.Log("Failed to remove test directory:", err) + } + t.testDir = "" +} + +func (t *T) Close() { + t.Log("close is no-op") +} + +// Printf implements Logger interfaces used by some libraries. +func (t *T) Printf(f string, a ...interface{}) { + t.Logf(f, a...) +} + +// Conn6 connects to the server listener at the specified named port using IPv6 loopback. +func (t *T) Conn6(portName string) Conn { + port := t.ports[portName] + if port == 0 { + panic("tests: connection for the unused port name is requested") + } + + conn, err := net.Dial("tcp6", "[::1]:"+strconv.Itoa(int(port))) + if err != nil { + t.Fatal("Could not connect, is server listening?", err) + } + + return Conn{ + T: t, + WriteTimeout: 1 * time.Second, + ReadTimeout: 15 * time.Second, + Conn: conn, + Scanner: bufio.NewScanner(conn), + } +} + +// Conn4 connects to the server listener at the specified named port using one +// of 127.0.0.0/8 addresses as a source. +func (t *T) Conn4(sourceIP, portName string) Conn { + port := t.ports[portName] + if port == 0 { + panic("tests: connection for the unused port name is requested") + } + + localIP := net.ParseIP(sourceIP) + if localIP == nil { + panic("tests: invalid localIP argument") + } + if localIP.To4() == nil { + panic("tests: only IPv4 addresses are allowed") + } + + conn, err := net.DialTCP("tcp4", &net.TCPAddr{ + IP: localIP, + Port: 0, + }, &net.TCPAddr{ + IP: net.IPv4(127, 0, 0, 1), + Port: int(port), + }) + if err != nil { + t.Fatal("Could not connect, is server listening?", err) + } + + return Conn{ + T: t, + WriteTimeout: 1 * time.Second, + ReadTimeout: 15 * time.Second, + Conn: conn, + Scanner: bufio.NewScanner(conn), + } +} + +var ( + DefaultSourceIP = net.IPv4(127, 109, 97, 100) + DefaultSourceIPRev = "100.97.109.127" +) + +func (t *T) ConnUnnamed(port uint16) Conn { + conn, err := net.DialTCP("tcp4", &net.TCPAddr{ + IP: DefaultSourceIP, + Port: 0, + }, &net.TCPAddr{ + IP: net.IPv4(127, 0, 0, 1), + Port: int(port), + }) + if err != nil { + t.Fatal("Could not connect, is server listening?", err) + } + + return Conn{ + T: t, + WriteTimeout: 1 * time.Second, + ReadTimeout: 15 * time.Second, + Conn: conn, + Scanner: bufio.NewScanner(conn), + } +} + +func (t *T) Conn(portName string) Conn { + port := t.ports[portName] + if port == 0 { + panic("tests: connection for the unused port name is requested") + } + + return t.ConnUnnamed(port) +} + +func (t *T) Subtest(name string, f func(t *T)) { + t.T.Run(name, func(subTT *testing.T) { + subT := *t + subT.T = subTT + f(&subT) + }) +} + +func init() { + flag.StringVar(&TestBinary, "integration.executable", "./maddy", "executable to test") + flag.StringVar(&CoverageOut, "integration.coverprofile", "", "write coverage stats to file (requires special maddy executable)") + flag.BoolVar(&DebugLog, "integration.debug", false, "pass -debug to maddy executable") +} diff --git a/tests/testdata/check_command.sh b/tests/testdata/check_command.sh new file mode 100755 index 0000000..4284b3c --- /dev/null +++ b/tests/testdata/check_command.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +if [ -e "${TEST_PWD}/testdata/${1}.hdr" ]; then + cat "${TEST_PWD}/testdata/${1}.hdr" +fi + +cat > ${TEST_STATE_DIR}/msg + +if [ -e "${TEST_PWD}/testdata/${1}.exit" ]; then + exit "$(cat "${TEST_PWD}/testdata/${1}.exit")" +fi diff --git a/tests/testdata/testing+addHeader@maddy.test.hdr b/tests/testdata/testing+addHeader@maddy.test.hdr new file mode 100644 index 0000000..71b07a4 --- /dev/null +++ b/tests/testdata/testing+addHeader@maddy.test.hdr @@ -0,0 +1 @@ +X-Added-Header: 1 diff --git a/tests/testdata/testing+reject@maddy.test.exit b/tests/testdata/testing+reject@maddy.test.exit new file mode 100644 index 0000000..48082f7 --- /dev/null +++ b/tests/testdata/testing+reject@maddy.test.exit @@ -0,0 +1 @@ +12