Merge main branch into next-node

.buildkite/pipeline.yml

@@ -1,6 +1,7 @@
 steps:
   - label: ':shipit: deploy devcontainer image for plutus-starter'
-    branches: "plutus-starter-devcontainer/v*"
+    # For example, "v0.1.0" will match, but not "plutus-starter-devcontainer/v0.1.0" or "v0.1.0-rc1"
+    branches: "v*.*.* !v*-*"
     command:
       - "./.buildkite/plutus-starter-devcontainer-push.sh"
     concurrency: 1

.github/workflows/test.yml

@@ -19,7 +19,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: nixbuild/nix-quick-install-action@v15
-      - run: nix-build -A tests.nixpkgsFmt -A tests.cabalFmt -A tests.purs-tidy -A tests.pngOptimization -A tests.shellcheck -A tests.stylishHaskell -A tests.generated --arg supportedSystems '[ builtins.currentSystem ]' --restrict-eval -I . --allowed-uris 'https://github.com/NixOS/nixpkgs https://github.com/input-output-hk https://github.com/NixOS/nixpkgs-channels' --option trusted-public-keys "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= iohk.cachix.org-1:DpRUyj7h7V830dp/i6Nti+NEO2/nhblbov/8MW7Rqoo= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" --option substituters "https://hydra.iohk.io https://iohk.cachix.org https://cache.nixos.org/"
+      - run: nix-build -A tests.nixpkgsFmt -A tests.cabalFmt -A tests.purs-tidy -A tests.pngOptimization -A tests.shellcheck -A tests.stylishHaskell --arg supportedSystems '[ builtins.currentSystem ]' --restrict-eval -I . --allowed-uris 'https://github.com/NixOS/nixpkgs https://github.com/input-output-hk https://github.com/NixOS/nixpkgs-channels' --option trusted-public-keys "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= iohk.cachix.org-1:DpRUyj7h7V830dp/i6Nti+NEO2/nhblbov/8MW7Rqoo= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" --option substituters "https://hydra.iohk.io https://iohk.cachix.org https://cache.nixos.org/"
   check-for-updates:
     strategy:
       matrix:

.gitignore

@@ -76,6 +76,8 @@ pkgs/.stack
 **/.spago/
 **/.spago2nix/
 **/.psa-stash
+plutus-playground-client/generated
+plutus-pab-executables/demo/pab-nami/client/generated
 
 # Backend config files
 marlowe-playground-server/playground.yaml

CONTRIBUTING.adoc

@@ -432,6 +432,8 @@ The documentation site is built on ReadTheDocs.
 It will build a preview for each PR which is linked from the PR status.
 It's useful to take a look if you're changing any of the documentation.
 
+Run `build-and-serve-docs` in nix-shell to host a local instance at http://0.0.0.0:8002/. Haddock is at http://0.0.0.0:8002/haddock.
+
 ==== Github Actions
 
 These perform some of the same checks as Hydra, but Github Actions is often more available, so they return faster and act as a "smoke check".

README.adoc

@@ -65,14 +65,20 @@ The core `plutus-apps` packages are versioned as follows:
 
 * Package versioning follows the https://pvp.haskell.org/[PVP] on a best-effort basis (i.e. we will generally try to but we won't guarantee it).
 ** The first-major-version component indicates the "era" which for our purposes means which major version of the *Cardano node* the tools are compatible with.
+*** Alonzo era: `v0.x.x`
+*** Babbage era: `v1.x.x`
 ** The second-major-version component is used for releases which are major versions according to the PVP, but which are still compatible with the current "era".
 ** The minor-version and below are used as normal.
 * Packages which are used downstream should all have the same version.
 * Other packages which are not used downstream (e.g. `plutus-playground-server`, `plutus-playground-client`, `quickcheck-dynamic`, etc.) can remain unversioned.
 
 In principle we could just have a single major version, but using two makes it easier to avoid mistakes and clearly expresses the state of the repository.
 
-=== Branching
+=== Branching and tagging
+
+The following tagging rules are followed:
+
+* Version `X` is tagged as `vX`
 
 There are two protected branches in `plutus-apps`:
 
@@ -82,7 +88,7 @@ There are two protected branches in `plutus-apps`:
 ** The version of transitive dependencies (`plutus`, `cardano-ledger`, `ouroboros-network`, etc.) should be pinned to the ones from `cardano-node` (or better, `cardano-wallet`)
 ** Changes will be backported form `main` to `next-node`
 * `next-node` branch: should always target the next node release.
-** This branch will eventually be merged in `main` after the Cardano mainnet HF.
+** This branch will eventually be merged in `main` after the Cardano mainnet HF and deleted. Once the next HF is planned, it will be recreated and it will contain an upgraded `cardano-node` version.
 
 === Dependency update
 

default.nix

@@ -43,7 +43,7 @@ rec {
       inherit (plutus-apps) purs-tidy;
       inherit (plutus-apps.lib) buildPursPackage buildNodeModules filterNpm gitignore-nix;
       inherit haskell webCommon;
-    }) client server start-backend generate-purescript generated-purescript;
+    }) client server start-backend generate-purescript;
   };
 
   # TODO: Fails for now because of webpack can't include `nami-wallet` lib in it's bundle.
@@ -53,7 +53,7 @@ rec {
       inherit (plutus-apps) purs-tidy;
       inherit pkgs haskell webCommon;
       inherit (plutus-apps.lib) buildPursPackage buildNodeModules filterNpm gitignore-nix;
-    }) client pab-setup-invoker pab-nami-demo-invoker pab-nami-demo-generator generate-purescript generated-purescript start-backend;
+    }) client pab-setup-invoker pab-nami-demo-invoker pab-nami-demo-generator start-backend;
   };
 
   plutus-use-cases = pkgs.recurseIntoAttrs (pkgs.callPackage ./plutus-use-cases {
@@ -66,14 +66,14 @@ rec {
 
   marconi = plutus-apps.haskell.packages.plutus-chain-index.components.exes.marconi;
 
+  create-script-context = plutus-apps.haskell.packages.plutus-example.components.exes.create-script-context;
+
   tests = import ./nix/tests/default.nix {
     inherit pkgs docs;
     inherit (plutus-apps.lib) gitignore-nix;
     inherit (plutus-apps) fixStylishHaskell fix-purs-tidy fixPngOptimization fixCabalFmt;
     inherit plutus-playground web-ghc;
     src = ./.;
-    play-generated = plutus-playground.generated-purescript;
-    nami-generated = pab-nami-demo.generated-purescript;
   };
 
   docs = import ./nix/docs.nix { inherit pkgs plutus-apps; };

doc/README.md

@@ -22,6 +22,8 @@ Or you can use Nix which will also build the Haddock for the project and link it
 nix build -f default.nix docs.site
 ```
 
+Or you can run `build-and-serve-docs` in nix-shell to host a local instance of ReadTheDocs at http://0.0.0.0:8002/. Haddock is at http://0.0.0.0:8002/haddock.
+
 The doc site from main is built automatically and hosted [here](https://plutus-apps.readthedocs.io/en/latest).
 
 Additionally, the site is built for all PRs, and a link to a preview can be found in the PR statuses.

doc/adr/0003-marconi-monorepo.rst

@@ -18,38 +18,37 @@ Draft
 Context
 -------
 
-Marconi is a Haskell executable that lives in `plutus-apps/plutus-chain-index`.
+Marconi is a Haskell executable and library that lives in `plutus-chain-index`.
 
 It is desirable to move it into a separate repository for the following reasons:
 
-* Better visibility and easier to discover.
-* It wants to depend on a version of `cardano-api` different to the one in 
-  `plutus-apps`.
-* It seems to "make sense" to warrant its own repository, as it is a fairly 
-  independent component.
+* Better visibility and easier to discover
+* It wants to update the version of its `cardano-api` dependency independently of the version used by `plutus-apps`
+* It is a farily independent component, therefore it warrants its own repository
 
 However, creating a separate repository would be rather costly.
-It would involve a great deal of duplication, due to the way our current 
-nix code is structured, not to mention the added complexity and overhead 
+It would involve a great deal of duplication, due to the way our current
+nix code is structured, not to mention the added complexity and overhead
 inherent in maintaining a separate codebase.
 
 Decision
 --------
 
-* Marconi will be kept in `plutus-apps` for the time being.
+* We will put Marconi in a separate Github repository
+* Until we resolve the issues with creating a separate Github repository (see Context), we will keep Marconi as a separate project in `plutus-apps`
 
 Implications
 ------------
 
-* A nix flake will be added in `plutus-apps` so that users will be able 
-  to obtain the executable trivially.
-* The possibility to specify a separate version of `cardano-api` just for 
-  marconi **whist staying in plutus-apps** will be explored.
-  Note that the dependency on a specific version `cardano-api` won't be a 
-  problem until the next hard fork.
-* As a very low priority task, a new repository *will* be created for marconi,
-  which will use `std` from the start 
+* A nix flake will be added in `plutus-apps` so that users will be able
+  to obtain the Marconi executable trivially
+* The possibility to specify a separate version of `cardano-api` just for
+  Marconi, **while staying in plutus-apps**, will be explored
+* As a very low priority task, a new repository *will* be created for Marconi,
+  which will use `std` from the start
   (see :ref:`Repository Standardization <repository_standardization>`)
 
+Related Decisions
+-----------------
 
-
+:ref:`Repository Standardization <repository_standardization>`

doc/requirements.txt

@@ -24,6 +24,7 @@ Jinja2==2.11.2
 jsonpointer==2.0
 jsonref==0.2
 MarkupSafe==1.1.1
+markdown==3.3.7
 Pygments==2.6.1
 pytz==2020.1
 requests==2.24.0

nix/devcontainer/deploy/default.nix

@@ -15,9 +15,10 @@ pkgs.writeScript "docker-build-push-devcontainer" ''
 
   tag="''${BUILDKITE_TAG:-}"
   echo "Git tag: ''${tag}."
-    
+
   # Pick out only the version component of a tag like:
   # "plutus-starter-devcontainer/v1.0" -> "v1.0"
+  # "v1.0" -> "v1.0"
   version="$(echo $tag | sed -e 's/.*[\/]//')"
 
   # Construct a tag to push up to dockerHub

nix/lib/purescript.nix

@@ -39,11 +39,11 @@ stdenv.mkDerivation {
     shopt -s globstar
     cp -R ${nodeModules}/node_modules .
     chmod -R u+rw ./node_modules
-    ${addExtraSrcs}
 
+    ${addExtraSrcs}
     install-spago-style
     echo building project...
-    psa --strict --censor-lib --stash --is-lib=$src/generated --is-lib=.spago ${spagoSources} ${extraPSPaths} "$src/**/*.purs"
+    psa --strict --censor-lib --stash --is-lib=generated --is-lib=.spago ${spagoSources} ${extraPSPaths} "$src/**/*.purs"
     echo done.
     npm run build:webpack:prod
   '';

nix/tests/default.nix

@@ -5,8 +5,6 @@
 , fixCabalFmt
 , fixPngOptimization
 , src
-, play-generated
-, nami-generated
 , plutus-playground
 , web-ghc
 , docs
@@ -17,18 +15,14 @@ let
   cleanSrc = gitignore-nix.gitignoreSource src;
 in
 pkgs.recurseIntoAttrs {
+
   shellcheck = pkgs.callPackage ./shellcheck.nix { src = cleanSrc; };
 
   stylishHaskell = pkgs.callPackage ./stylish-haskell.nix {
     src = cleanSrc;
     inherit fixStylishHaskell;
   };
 
-  generated = pkgs.callPackage ./generated.nix {
-    src = cleanSrc;
-    inherit play-generated nami-generated;
-  };
-
   purs-tidy = pkgs.callPackage ./purs-tidy.nix {
     src = cleanSrc;
     inherit fix-purs-tidy;

playground-common/src/PSGenerator/Common.hs

@@ -36,7 +36,7 @@ import Ledger.TimeSlot (SlotConfig, SlotConversionError)
 import Ledger.Tx.CardanoAPI (FromCardanoError, ToCardanoError)
 import Ledger.Value (AssetClass, CurrencySymbol, TokenName, Value)
 import Playground.Types (ContractCall, FunctionSchema, KnownCurrency)
-import Plutus.ChainIndex.Api (IsUtxoResponse, TxosResponse, UtxosResponse)
+import Plutus.ChainIndex.Api (IsUtxoResponse, QueryResponse, TxosResponse, UtxosResponse)
 import Plutus.ChainIndex.ChainIndexError (ChainIndexError)
 import Plutus.ChainIndex.ChainIndexLog (ChainIndexLog)
 import Plutus.ChainIndex.Tx (ChainIndexTx, ChainIndexTxOutputs, ReferenceScript)
@@ -417,6 +417,7 @@ ledgerTypes =
     , equal . genericShow . argonaut $ mkSumType @ChainIndexQuery
     , equal . genericShow . argonaut $ mkSumType @ChainIndexResponse
     , equal . genericShow . argonaut $ mkSumType @IsUtxoResponse
+    , equal . genericShow . argonaut $ mkSumType @(QueryResponse A)
     , equal . genericShow . argonaut $ mkSumType @TxosResponse
     , equal . genericShow . argonaut $ mkSumType @UtxosResponse
     , equal . genericShow . argonaut $ mkSumType @ChainIndexTx

plutus-chain-index-core/src/Plutus/ChainIndex/Api.hs

@@ -17,10 +17,14 @@ module Plutus.ChainIndex.Api
   , swagger
   , TxoAtAddressRequest(..)
   , TxosResponse(..)
+  , QueryAtAddressRequest (..)
+  , QueryResponse(..)
+  , collectQueryResponse
   ) where
 
 import Control.Monad.Freer.Extras.Pagination (Page, PageQuery)
 import Data.Aeson (FromJSON, ToJSON, Value)
+import Data.Default (def)
 import Data.OpenApi qualified as OpenApi
 import Data.Proxy (Proxy (..))
 import GHC.Generics (Generic)
@@ -144,6 +148,26 @@ data TxosResponse = TxosResponse
     }
     deriving (Show, Eq, Generic, FromJSON, ToJSON, OpenApi.ToSchema)
 
+
+data QueryAtAddressRequest = QueryAtAddressRequest
+    { pageQuery  :: Maybe (PageQuery TxOutRef)
+    , credential :: Credential
+    }
+    deriving (Show, Eq, Generic, FromJSON, ToJSON, OpenApi.ToSchema)
+
+-- | generic response type endpoint
+-- This type is introduced to avoid querying the chain index twice to obtain the expected info.
+-- Indeed, it returns the next page query if more items are available
+data QueryResponse a = QueryResponse
+    { queryResult :: a
+    , nextQuery   :: Maybe (PageQuery TxOutRef)
+    }
+    deriving (Show, Generic, Eq)
+
+deriving instance (FromJSON a, Generic a) => FromJSON (QueryResponse a)
+deriving instance (ToJSON a, Generic a) => ToJSON (QueryResponse a)
+deriving instance (OpenApi.ToSchema a, Generic a) => OpenApi.ToSchema (QueryResponse a)
+
 type API
     = "healthcheck" :> Description "Is the server alive?" :> Get '[JSON] NoContent
     :<|> "from-hash" :> FromHashAPI
@@ -152,6 +176,7 @@ type API
     :<|> "tx" :> Description "Get a transaction from its id." :> ReqBody '[JSON] TxId :> Post '[JSON] ChainIndexTx
     :<|> "is-utxo" :> Description "Check if the reference is an UTxO." :> ReqBody '[JSON] TxOutRef :> Post '[JSON] IsUtxoResponse
     :<|> "utxo-at-address" :> Description "Get all UTxOs at an address." :> ReqBody '[JSON] UtxoAtAddressRequest :> Post '[JSON] UtxosResponse
+    :<|> "unspent-txouts-at-address" :> Description "Get all unspent transaction output at an address." :> ReqBody '[JSON] QueryAtAddressRequest :> Post '[JSON] (QueryResponse [(TxOutRef, ChainIndexTxOut)])
     :<|> "utxo-with-currency" :> Description "Get all UTxOs with a currency." :> ReqBody '[JSON] UtxoWithCurrencyRequest :> Post '[JSON] UtxosResponse
     :<|> "txs" :> Description "Get transactions from a list of their ids." :> ReqBody '[JSON] [TxId] :> Post '[JSON] [ChainIndexTx]
     :<|> "txo-at-address" :> Description "Get TxOs at an address." :> ReqBody '[JSON] TxoAtAddressRequest :> Post '[JSON] TxosResponse
@@ -173,3 +198,15 @@ swagger = swaggerSchemaUIServer (toOpenApi (Proxy @API))
 
 -- We don't include `SwaggerAPI` into `API` to exclude it from the effects code.
 type FullAPI = API :<|> SwaggerAPI
+
+-- | Go through each 'Page's of 'QueryResponse', and collect the results.
+collectQueryResponse ::
+    ( Monad m )
+    => (PageQuery TxOutRef -> m (QueryResponse a)) -- ^ query response function
+    -> m [a]
+collectQueryResponse q = go (Just def)
+  where
+    go Nothing = pure []
+    go (Just pq) = do
+      res <- q pq
+      (queryResult res :) <$> go (nextQuery res)