diff --git a/.travis.yml b/.travis.yml index 27b0ea7..b14ad69 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,7 @@ script: | ln -s curl-runnings-${version}.tar.gz curl-runnings.tar.gz && mkdir -p $HOME/cr-release && cd $HOME/cr-release && - ln -s $HOME/.local/bin/curl-runnings-${version}.tar.gz curl-runnings-${version}.tar.gz + ln -s $HOME/.local/bin/curl-runnings-${version}.tar.gz curl-runnings-${version}-linux.tar.gz cache: directories: - $HOME/.stack diff --git a/README.md b/README.md index 8059400..73a3dd9 100644 --- a/README.md +++ b/README.md @@ -6,36 +6,19 @@ _Feel the rhythm! Feel the rhyme! Get on up, it's testing time! curl-runnings!_ -curl-runnings is a framework for writing declarative, curl based tests for your -APIs. Write your tests quickly and correctly with a straight-forward -specification in yaml or json that can encode simple but powerful matchers -against responses. +A common form of black-box API testing boils down to simply making requests to +an endpoint and verifying properties of the response. curl-runnings aims to make +writing tests like this fast and easy. + +curl-runnings is a framework for writing declarative tests for your APIs in a +fashion equivalent to performing `curl`s and verifying the responses. Write your +tests quickly and correctly with a straight-forward specification in +[Dhall](https://dhall-lang.org/), yaml, or json that can encode simple but +powerful matchers against responses. Alternatively, you can use the curl-runnings library to write your tests in Haskell (though a Haskell setup is absolutely not required to use this tool). -### Why? - -This library came out of a pain-point my coworkers and I were running into -during development: Writing integration tests for our APIs was generally -annoying. They were time consuming to write especially considering how basic -they were, and we are a small startup where developer time is in short supply. -Over time, we found ourselves sometimes just writing bash scripts that would -`curl` our various endpoints and check the output with very basic matchers. -These tests were fast to write, but quickly became difficult to maintain as -complexity was added. Not only did maintenance become challenging, but the whole -system was very error prone and confidence in the tests overall was decreasing. -At the end of the day, we needed to just curl some endpoints and verify the -output looks sane, and do this quickly and correctly. This is precisely the goal -of curl-runnings. - -Now you can write your tests just as data in a yaml or json file, -and curl-runnings will take care of the rest! - -While yaml/json is the current way to write curl-runnings tests, this project is -being built in a way that should lend itself well to an embedded domain specific -language, which is a future goal for the project. curl-runnings specs in Dhall -is also being developed and may fulfill the same needs. ### Installing @@ -55,10 +38,31 @@ Alternatively, you can compile from source with stack. Curl runnings tests are just data! A test spec is an object containing an array of `cases`, where each item represents a single curl and set of assertions about -the response. Write your tests specs in a yaml or json file. Note: the legacy +the response. Write your tests specs in a Dhall, yaml or json file. Note: the legacy format of a top level array of test cases is still supported, but may not be in future releases. +```dhall +let JSON = https://prelude.dhall-lang.org/JSON/package.dhall + +let CurlRunnings = ./dhall/curl-runnings.dhall + +in CurlRunnings.hydrateCase + CurlRunnings.Case::{ + , expectData = Some + ( CurlRunnings.ExpectData.Exactly + ( JSON.object + [ { mapKey = "okay", mapValue = JSON.bool True }, + { mapKey = "message", mapValue = JSON.string "a message" }] + ) + ) + , expectStatus = 200 + , name = "test 1" + , requestMethod = CurlRunnings.HttpMethod.GET + , url = "http://your-endpoing.com/status" + } +``` + ```yaml --- diff --git a/dhall/curl-runnings.dhall b/dhall/curl-runnings.dhall new file mode 100644 index 0000000..a752c88 --- /dev/null +++ b/dhall/curl-runnings.dhall @@ -0,0 +1,240 @@ +let JSON = https://prelude.dhall-lang.org/JSON/package.dhall + +let List/map = https://prelude.dhall-lang.org/List/map + +let Optional/map = https://prelude.dhall-lang.org/Optional/map + +let Map = https://prelude.dhall-lang.org/Map/Type + +let HttpMethod + : Type + = < GET | POST | PUT | PATCH | DELETE > + +let PartialMatcher = + < KeyMatch : Text + | ValueMatch : JSON.Type + | KeyValueMatch : { key : Text, value : JSON.Type } + > + +let ExpectData = + < Exactly : JSON.Type + | Contains : List PartialMatcher + | NotContains : List PartialMatcher + | MixedContains : + { contains : List PartialMatcher, notContains : List PartialMatcher } + > + +let KeyValMatchHydrated = + { keyMatch : Optional PartialMatcher + , valueMatch : Optional PartialMatcher + , keyValueMatch : Optional PartialMatcher + } + +let ExpectHeaders = + < HeaderString : Text + | HeaderKeyVal : { key : Optional Text, value : Optional Text } + > + +let hydrateContains = + λ(containsMatcher : PartialMatcher) → + merge + { KeyMatch = + λ(k : Text) → + { keyMatch = Some (PartialMatcher.KeyMatch k) + , valueMatch = None PartialMatcher + , keyValueMatch = None PartialMatcher + } + , ValueMatch = + λ(v : JSON.Type) → + { keyMatch = None PartialMatcher + , valueMatch = Some (PartialMatcher.ValueMatch v) + , keyValueMatch = None PartialMatcher + } + , KeyValueMatch = + λ(args : { key : Text, value : JSON.Type }) → + { keyMatch = None PartialMatcher + , valueMatch = None PartialMatcher + , keyValueMatch = Some + ( PartialMatcher.KeyValueMatch + { key = args.key, value = args.value } + ) + } + } + containsMatcher + +let ExpectResponseHydrated = + { exactly : Optional ExpectData + , contains : Optional (List KeyValMatchHydrated) + , notContains : Optional (List KeyValMatchHydrated) + } + +let hydrateExpectData = + λ(matcher : ExpectData) → + merge + { Exactly = + λ(j : JSON.Type) → + { exactly = Some (ExpectData.Exactly j) + , contains = None (List KeyValMatchHydrated) + , notContains = None (List KeyValMatchHydrated) + } + , Contains = + λ(ms : List PartialMatcher) → + { exactly = None ExpectData + , contains = Some + ( List/map + PartialMatcher + KeyValMatchHydrated + hydrateContains + ms + ) + , notContains = None (List KeyValMatchHydrated) + } + , NotContains = + λ(ms : List PartialMatcher) → + { exactly = None ExpectData + , contains = None (List KeyValMatchHydrated) + , notContains = Some + ( List/map + PartialMatcher + KeyValMatchHydrated + hydrateContains + ms + ) + } + , MixedContains = + λ ( args + : { contains : List PartialMatcher + , notContains : List PartialMatcher + } + ) → + { exactly = None ExpectData + , contains = Some + ( List/map + PartialMatcher + KeyValMatchHydrated + hydrateContains + args.contains + ) + , notContains = Some + ( List/map + PartialMatcher + KeyValMatchHydrated + hydrateContains + args.notContains + ) + } + } + matcher + +let BodyType = < json | urlencoded > + +let RequestData = < JSON : JSON.Type | UrlEncoded : Map Text Text > + +let RequestDataHydrated = { bodyType : BodyType, content : RequestData } + +let hydrateRquestData = + λ(reqData : RequestData) → + merge + { JSON = + λ(json : JSON.Type) → + { bodyType = BodyType.json, content = RequestData.JSON json } + , UrlEncoded = + λ(encoded : Map Text Text) → + { bodyType = BodyType.urlencoded + , content = RequestData.UrlEncoded encoded + } + } + reqData + +let makeQueryParams = + λ(params : Map Text Text) → + JSON.object + ( List/map + { mapKey : Text, mapValue : Text } + { mapKey : Text, mapValue : JSON.Type } + ( λ(args : { mapKey : Text, mapValue : Text }) → + { mapKey = args.mapKey, mapValue = JSON.string args.mapValue } + ) + params + ) + +let QueryParams = List { mapKey : Text, mapValue : JSON.Type } + +let Case = + { Type = + { name : Text + , url : Text + , requestMethod : HttpMethod + , queryParameters : Map Text Text + , expectData : Optional ExpectData + , expectStatus : Natural + , headers : Optional Text + , expectHeaders : Optional (List ExpectHeaders) + , allowedRedirects : Natural + , requestData : Optional RequestData + } + , default = + { expectData = None ExpectData + , headers = None Text + , expectHeaders = None (List ExpectHeaders) + , allowedRedirects = 10 + , queryParameters = [] : Map Text Text + , requestData = None RequestData + } + } + +let HydratedCase = + { Type = + { name : Text + , url : Text + , requestMethod : HttpMethod + , queryParameters : JSON.Type + , expectData : Optional ExpectResponseHydrated + , expectStatus : Natural + , headers : Optional Text + , expectHeaders : Optional (List ExpectHeaders) + , allowedRedirects : Natural + , requestData : Optional RequestDataHydrated + } + , default = + { expectData = None ExpectResponseHydrated + , headers = None Text + , expectHeaders = None (List ExpectHeaders) + , allowedRedirects = 10 + , queryParameters = JSON.null + , requestData = None RequestDataHydrated + } + } + +let hydrateCase = + λ(c : Case.Type) → + c + ⫽ { queryParameters = makeQueryParams c.queryParameters + , expectData = + Optional/map + ExpectData + ExpectResponseHydrated + hydrateExpectData + c.expectData + , requestData = + Optional/map + RequestData + RequestDataHydrated + hydrateRquestData + c.requestData + } + +let hydrateCases = + λ(cases : List Case.Type) → + List/map Case.Type HydratedCase.Type hydrateCase cases + +in { Case + , HydratedCase + , hydrateCase + , hydrateCases + , HttpMethod + , ExpectData + , PartialMatcher + , ExpectHeaders + , RequestData + } diff --git a/examples/example-spec.dhall b/examples/example-spec.dhall new file mode 100644 index 0000000..3ea86cd --- /dev/null +++ b/examples/example-spec.dhall @@ -0,0 +1,97 @@ +-- Your curl-runnings specs can be written in dhall, which can give you great +-- type safety and interpolation abilities. + +-- The curl-runnings dhall module offers some of the types and functions that +-- can give you extra safety, or you can target the json specification directly +-- if you prefer. You can import the dhall module directly or via url: +-- https://raw.githubusercontent.com/aviaviavi/curl-runnings/master/dhall/curl-runnings.dhall +let JSON = https://prelude.dhall-lang.org/JSON/package.dhall + +let CurlRunnings = ./dhall/curl-runnings.dhall + +let List/map = https://prelude.dhall-lang.org/List/map + +let Optional/map = https://prelude.dhall-lang.org/Optional/map + +let Map = https://prelude.dhall-lang.org/Map/Type + +let host = "https://tabdextension.com" + +in CurlRunnings.hydrateCases + [ CurlRunnings.Case::{ + , expectData = Some + ( CurlRunnings.ExpectData.Exactly + ( JSON.object + [ { mapKey = "ping", mapValue = JSON.string "pong" } ] + ) + ) + , expectStatus = 200 + , name = "test 1" + , requestMethod = CurlRunnings.HttpMethod.GET + , url = host ++ "/ping" + } + , CurlRunnings.Case::{ + , expectData = Some + ( CurlRunnings.ExpectData.Contains + [ CurlRunnings.PartialMatcher.KeyMatch "ping" + , CurlRunnings.PartialMatcher.ValueMatch (JSON.string "pong") + , CurlRunnings.PartialMatcher.KeyValueMatch + { key = "ping", value = JSON.string "pong" } + ] + ) + , expectStatus = 200 + , name = "test 2" + , requestMethod = CurlRunnings.HttpMethod.GET + , url = host ++ "/ping" + } + , CurlRunnings.Case::{ + , expectData = Some + ( CurlRunnings.ExpectData.NotContains + [ CurlRunnings.PartialMatcher.KeyMatch "poing" + , CurlRunnings.PartialMatcher.ValueMatch (JSON.string "pongg") + , CurlRunnings.PartialMatcher.KeyValueMatch + { key = "ping", value = JSON.string "poong" } + ] + ) + , expectStatus = 200 + , name = "test 3" + , requestMethod = CurlRunnings.HttpMethod.GET + , url = host ++ "/ping" + } + , CurlRunnings.Case::{ + , expectData = Some + ( CurlRunnings.ExpectData.MixedContains + { contains = + [ CurlRunnings.PartialMatcher.KeyMatch "ping" + , CurlRunnings.PartialMatcher.ValueMatch + (JSON.string "\$") + , CurlRunnings.PartialMatcher.KeyValueMatch + { key = "ping", value = JSON.string "pong" } + ] + , notContains = + [ CurlRunnings.PartialMatcher.KeyMatch "poing" + , CurlRunnings.PartialMatcher.ValueMatch (JSON.string "pongg") + , CurlRunnings.PartialMatcher.KeyValueMatch + { key = "ping", value = JSON.string "poong" } + ] + } + ) + , expectStatus = 200 + , name = "test 4" + , requestMethod = CurlRunnings.HttpMethod.GET + , url = host ++ "/ping" + } + , CurlRunnings.Case::{ + , expectStatus = 405 + , name = "test 5" + , requestMethod = CurlRunnings.HttpMethod.POST + , url = host ++ "/ping" + , queryParameters = [ { mapKey = "test", mapValue = "asdf" } ] + , requestData = Some + ( CurlRunnings.RequestData.JSON + ( JSON.object + [ { mapKey = "key", mapValue = JSON.string "value" } ] + ) + ) + } + ] diff --git a/package.yaml b/package.yaml index cbc802c..847e826 100644 --- a/package.yaml +++ b/package.yaml @@ -1,10 +1,10 @@ name: curl-runnings -version: 0.15.0 +version: 0.16.0 github: aviaviavi/curl-runnings license: MIT author: Avi Press maintainer: mail@avi.press -copyright: 2018 Avi Press +copyright: 2020 Avi Press category: Testing synopsis: A framework for declaratively writing curl based API tests @@ -33,8 +33,11 @@ library: - case-insensitive >=0.2.1 - base64-bytestring >=1.0.0.2 - clock >=0.7.2 + - dhall >=1.8.2 + - dhall-json >= 1.0.9 - directory >=1.3.0.2 - - hspec >= 2.4.4 + - hashable >= 1.3.0.0 + - hspec >=2.4.4 - hspec-expectations >=0.8.2 - http-conduit >=2.3.6 - megaparsec >=7.0.4 @@ -45,6 +48,7 @@ library: - regex-posix >=0.95.2 - text >=1.2.2.2 - time >=1.8.0.2 + - transformers >=0.5.2.0 - unordered-containers >=0.2.8.0 - vector >=0.12.0 - yaml >=0.8.28 diff --git a/src/Testing/CurlRunnings.hs b/src/Testing/CurlRunnings.hs index 17c4388..e5aa516 100644 --- a/src/Testing/CurlRunnings.hs +++ b/src/Testing/CurlRunnings.hs @@ -13,8 +13,12 @@ module Testing.CurlRunnings , decodeFile ) where +import Control.Arrow import Control.Exception +import qualified Control.Exception import Control.Monad +import Control.Monad.IO.Class +import Control.Monad.Trans.Except import Data.Aeson import Data.Aeson.Types import qualified Data.ByteString.Base64 as B64 @@ -28,9 +32,19 @@ import Data.Maybe import Data.Monoid import qualified Data.Text as T import qualified Data.Text.Encoding as T +import qualified Data.Text.IO as TIO +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Lazy.IO as TLIO import Data.Time.Clock import qualified Data.Vector as V +import qualified Data.Vector as V +import qualified Data.Yaml as Y import qualified Data.Yaml.Include as YI +import qualified Dhall +import qualified Dhall.Import +import qualified Dhall.JSON +import qualified Dhall.Parser +import qualified Dhall.TypeCheck import Network.Connection (TLSSettings (..)) import Network.HTTP.Client.TLS (mkManagerSettings) import Network.HTTP.Conduit @@ -45,8 +59,7 @@ import Testing.CurlRunnings.Types import Text.Printf import Text.Regex.Posix - --- | decode a json or yaml file into a suite object +-- | decode a json, yaml, or dhall file into a suite object decodeFile :: FilePath -> IO (Either String CurlSuite) decodeFile specPath = doesFileExist specPath >>= \exists -> @@ -56,9 +69,31 @@ decodeFile specPath = eitherDecode' <$> B.readFile specPath :: IO (Either String CurlSuite) "yaml" -> mapLeft show <$> YI.decodeFileEither specPath "yml" -> mapLeft show <$> YI.decodeFileEither specPath + "dhall" -> do + runExceptT $ do + let showErrorWithMessage :: (Show a) => String -> a -> String + showErrorWithMessage message err = message ++ ": " ++ (show err) + raw <- liftIO $ TIO.readFile specPath + expr <- + withExceptT (showErrorWithMessage "parser") . ExceptT . return $ + Dhall.Parser.exprFromText "dhall parser" (raw :: Dhall.Text) + expr' <- liftIO $ Dhall.Import.load expr + ExceptT $ + return $ do + _ <- + left (showErrorWithMessage "typeof") $ + Dhall.TypeCheck.typeOf expr' + val <- + left (showErrorWithMessage "to json") $ + Dhall.JSON.dhallToJSON expr' + left (showErrorWithMessage "from json") . resultToEither $ + fromJSON val _ -> return . Left $ printf "Invalid spec path %s" specPath else return . Left $ printf "%s not found" specPath +resultToEither :: Result a -> Either String a +resultToEither (Error s) = Left s +resultToEither (Success a) = Right a noVerifyTlsManagerSettings :: ManagerSettings noVerifyTlsManagerSettings = mkManagerSettings noVerifyTlsSettings Nothing diff --git a/src/Testing/CurlRunnings/Internal.hs b/src/Testing/CurlRunnings/Internal.hs index 5131a3a..21be240 100644 --- a/src/Testing/CurlRunnings/Internal.hs +++ b/src/Testing/CurlRunnings/Internal.hs @@ -71,7 +71,7 @@ type CurlRunningsLogger = (LogLevel -> T.Text -> IO ()) type CurlRunningsUnsafeLogger a = (LogLevel -> T.Text -> a -> a) makeLogger :: LogLevel -> CurlRunningsLogger -makeLogger threshold level text = when (level <= threshold) $ P.pPrint text +makeLogger threshold level text = when (level <= threshold) $ putStrLn $ T.unpack text makeUnsafeLogger :: Show a => LogLevel -> CurlRunningsUnsafeLogger a makeUnsafeLogger threshold level text object = diff --git a/src/Testing/CurlRunnings/Types.hs b/src/Testing/CurlRunnings/Types.hs index 0a80159..34a4d7a 100644 --- a/src/Testing/CurlRunnings/Types.hs +++ b/src/Testing/CurlRunnings/Types.hs @@ -41,6 +41,7 @@ module Testing.CurlRunnings.Types import Data.Aeson import Data.Aeson.Types import qualified Data.Char as C +import Data.Hashable (Hashable) import qualified Data.HashMap.Strict as H import Data.Maybe import Data.Monoid @@ -82,15 +83,19 @@ instance ToJSON JsonMatcher instance FromJSON JsonMatcher where parseJSON (Object v) - | isJust $ H.lookup "exactly" v = Exactly <$> v .: "exactly" - | isJust (H.lookup "contains" v) && isJust (H.lookup "notContains" v) = do - c <- Contains <$> v .: "contains" - n <- NotContains <$> v .: "notContains" - return $ MixedContains [c, n] - | isJust $ H.lookup "contains" v = Contains <$> v .: "contains" - | isJust $ H.lookup "notContains" v = NotContains <$> v .: "notContains" + | justAndNotEmpty "exactly" v = Exactly <$> v .: "exactly" + | justAndNotEmpty "contains" v && justAndNotEmpty "notContains" v = do + c <- Contains <$> v .: "contains" + n <- NotContains <$> v .: "notContains" + return $ MixedContains [c, n] + | justAndNotEmpty "contains" v = Contains <$> v .: "contains" + | justAndNotEmpty "notContains" v = NotContains <$> v .: "notContains" parseJSON invalid = typeMismatch "JsonMatcher" invalid +justAndNotEmpty :: (Eq k, Hashable k) => k -> H.HashMap k Value -> Bool +justAndNotEmpty key obj = + (isJust $ H.lookup key obj) && (H.lookup key obj /= Just Null) + -- | Simple predicate to check value constructor type isContains :: JsonMatcher -> Bool isContains (Contains _) = True @@ -171,18 +176,19 @@ data JsonSubExpr instance FromJSON JsonSubExpr where parseJSON (Object v) - | isJust $ H.lookup "keyValueMatch" v = + | justAndNotEmpty "keyValueMatch" v = let toParse = fromJust $ H.lookup "keyValueMatch" v in case toParse of Object o -> KeyValueMatch <$> o .: "key" <*> o .: "value" _ -> typeMismatch "JsonSubExpr" toParse - | isJust $ H.lookup "keyMatch" v = + | justAndNotEmpty "keyMatch" v = let toParse = fromJust $ H.lookup "keyMatch" v in case toParse of String s -> return $ KeyMatch s _ -> typeMismatch "JsonSubExpr" toParse - | isJust $ H.lookup "valueMatch" v = ValueMatch <$> v .: "valueMatch" + | justAndNotEmpty "valueMatch" v = ValueMatch <$> v .: "valueMatch" parseJSON invalid = typeMismatch "JsonSubExpr" invalid + instance ToJSON JsonSubExpr -- | Check the status code of a response. You can specify one or many valid codes. diff --git a/stack.yaml b/stack.yaml index 58124fd..baa1271 100644 --- a/stack.yaml +++ b/stack.yaml @@ -15,7 +15,7 @@ # resolver: # name: custom-snapshot # location: "./custom-snapshot.yaml" -resolver: lts-13.13 +resolver: lts-16.16 # User packages to be built. # Various formats can be used as shown in the example below. diff --git a/test/Spec.hs b/test/Spec.hs index 8993fb1..760dc90 100644 --- a/test/Spec.hs +++ b/test/Spec.hs @@ -28,6 +28,9 @@ main = hspec $ it "should provide valid example json specs" $ testValidSpec "/examples/example-spec.json" + it "should provide valid example dhall specs" $ + testValidSpec "/examples/example-spec.dhall" + -- note that this doesn't actually try to parse the interpolations themselves, -- but that would be useful thing to add here it "should provid a valid interpolation spec" $