updates outdated etherscan txn find/replace functionality to use blocknative

app/app/settings.py

@@ -63,6 +63,7 @@
 RATELIMIT_ENABLE = env.bool('RATELIMIT_ENABLE', default=True)
 RATELIMIT_USE_CACHE = env('RATELIMIT_USE_CACHE', default='default')
 RATELIMIT_VIEW = env('RATELIMIT_VIEW', default='tdi.views.ratelimited')
+BLOCKNATIVE_API = env('BLOCKNATIVE_API', default='')
 
 ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', default=['*'])
 CSRF_TRUSTED_ORIGINS = env.list('CSRF_TRUSTED_ORIGINS', default=['localhost'])

app/app/urls.py

@@ -680,6 +680,7 @@
     re_path(r'^_administration/viz/calendar/(.*)?$', dataviz.d3_views.viz_calendar, name='viz_calendar'),
     re_path(r'^_administration/viz/draggable/(.*)?$', dataviz.d3_views.viz_draggable, name='viz_draggable'),
     re_path(r'^_administration/viz/scatterplot/(.*)?$', dataviz.d3_views.viz_scatterplot, name='viz_scatterplot'),
+    url(r'^blocknative', perftools.views.blocknative, name='blocknative'),
 
     # for robots
     url(r'^robots.txt/?', retail.views.robotstxt, name='robotstxt'),

app/dashboard/models.py

@@ -1762,6 +1762,10 @@ def update_receive_tx_status(self):
 
         """
         from dashboard.utils import get_tx_status
+        from economy.tx import getReplacedTX
+        new_receive_txid = getReplacedTX(self.receive_txid)
+        if new_receive_txid:
+            self.receive_txid = new_receive_txid
         self.receive_tx_status, self.receive_tx_time = get_tx_status(self.receive_txid, self.network, self.created_on)
         return bool(self.receive_tx_status)
 
@@ -2035,7 +2039,7 @@ def psave_bounty_fulfilll(sender, instance, **kwargs):
                 "value_usd":instance.bounty.value_in_usdt_then,
                 "url":instance.bounty.url,
                 "network":instance.bounty.network,
-                "txid":'',
+                "txid": instance.payout_tx_id,
                 "token_name":instance.bounty.token_name,
                 "token_value":instance.bounty.value_in_token,
             }
@@ -5274,6 +5278,10 @@ def post_save_earning(sender, instance, created, **kwargs):
     if created:
         instance.create_auto_follow()
 
+        from economy.utils import watch_txn
+        if instance.txid:
+            watch_txn(instance.txid)
+
 def get_my_earnings_counter_profiles(profile_pk):
     # returns profiles that a user has done business with
     from_profile_earnings = Earning.objects.filter(from_profile=profile_pk)

app/dashboard/sync/eth.py

@@ -65,6 +65,10 @@ def get_eth_txn_status(fulfillment):
 
 def sync_eth_payout(fulfillment):
     if fulfillment.payout_tx_id:
+        from economy.tx import getReplacedTX
+        replacement_payout_tx_id = fulfillment.payout_tx_id
+        if replacement_payout_tx_id:
+            fulfillment.payout_tx_id = replacement_payout_tx_id
         txn_status = get_eth_txn_status(fulfillment)
         if txn_status:
             if txn_status.get('status') == 'done':

app/economy/admin.py

@@ -22,7 +22,7 @@
 from django.contrib import admin
 from django.utils.html import format_html
 
-from .models import ConversionRate, Token
+from .models import ConversionRate, Token, TXUpdate
 
 
 class TokenAdmin(admin.ModelAdmin):
@@ -48,7 +48,14 @@ def response_change(self, request, obj):
         return redirect(obj.admin_url)
 
 
-# Register your models here.
+class TXUpdateAdmin(admin.ModelAdmin):
+    """Handle displaying conversion rates in the django admin."""
+
+    ordering = ['-id']
+    search_fields = ['body']
+    list_display = ['id', 'created_on', 'processed', '__str__']
+
+
 class ConvRateAdmin(admin.ModelAdmin):
     """Handle displaying conversion rates in the django admin."""
 
@@ -59,3 +66,4 @@ class ConvRateAdmin(admin.ModelAdmin):
 
 admin.site.register(ConversionRate, ConvRateAdmin)
 admin.site.register(Token, TokenAdmin)
+admin.site.register(TXUpdate, TXUpdateAdmin)

app/economy/migrations/0004_txupdate.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.2.4 on 2020-09-21 20:05
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import economy.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('economy', '0003_auto_20200630_1304'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TXUpdate',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_on', models.DateTimeField(db_index=True, default=economy.models.get_time)),
+                ('modified_on', models.DateTimeField(default=economy.models.get_time)),
+                ('body', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, null=True)),
+                ('processed', models.BooleanField(default=False)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

app/economy/models.py

@@ -208,6 +208,32 @@ def reverse_conversion_rate(sender, instance, **kwargs):
         )
 
 
+class TXUpdate(SuperModel):
+    """Define the TXUpdate model."""
+
+    body = JSONField(null=True, default=dict, blank=True)
+    processed = models.BooleanField(default=False)
+
+    def __str__(self):
+        """Define the string representation of this object."""
+        return f"{self.body['hash']}"
+
+    def process_callbacks(self):
+        tx_id = self.body['hash']
+        status = self.body.get('status')
+        replaceHash = self.body.get('replaceHash')
+        if status:
+            # https://docs.blocknative.com/webhook-api#transaction-events-state-changes
+            # TODO: move processing of failed, dropped, txns from update_tx_status to here
+            pass
+        if replaceHash:
+            # TODO: find/replace any txids that are wrong here
+            # example replacement payload
+            # {"r": "0x667f3b3c35715598bc244a8fef7ca432eb725806571c99c4dca00454d08ee4ce", "s": "0x67de52717d0eef915a33250c5a849458f44215e79f7df1ea0867035c0359c5e3", "v": "0x25", "to": "0x00De4B13153673BCAE2616b67bf822500d325Fc3", "gas": 21000, "from": "0x00De4B13153673BCAE2616b67bf822500d325Fc3", "hash": "0x797a61f94f7fb20528fdbedb9d28ebdedd721ad5a2b0f4d4ac991f2ca1dcfc9b", "asset": "ETH", "input": "0x", "nonce": 4154, "value": "0", "apiKey": "d1ff5891-98f0-4324-9541-e56f54252149", "status": "speedup", "system": "ethereum", "network": "main", "gasPrice": "60000000000", "blockHash": null, "monitorId": "GETH_1_C_PROD", "timeStamp": "2020-09-21T20:27:33.924Z", "blockNumber": null, "replaceHash": "0x16903550083ee036077e67539f5da0c3bea44d874845083007cdd934e152e015", "timePending": "46956", "serverVersion": "0.64.3", "monitorVersion": "0.69.1"}
+            pass
+        self.processed = True
+        return
+
 class Token(SuperModel):
     """Define the Token model."""
 

app/economy/tx.py

@@ -51,20 +51,14 @@ class TransactionNotFound(Exception):
 get_decimals = lambda contract : int(contract.functions.decimals().call())
 
 
-# scrapes etherscan to get the replaced tx
 def getReplacedTX(tx):
-    response = requests.get(ethurl + tx + '/', headers=headers)
-    soup = BeautifulSoup(response.content, "html.parser")
-    # look for span that contains the dropped&replaced msg
-    p = soup.find("span", "u-label u-label--sm u-label--warning rounded")
-    if not p:
-        return None
-    if "Replaced" in p.text:  # check if it's a replaced tx
-        # get the id for the replaced tx
-        q = soup.find(href=re.compile("/tx/0x"))
-        return q.text
-    else:
-        return None
+    from economy.models import TXUpdate
+    txus = TXUpdate.objects.filter(body__hash__iexact=tx)
+    for txu in txus:
+        replace_hash = txu.body.get('replaceHash')
+        if replace_hash:
+            return replace_hash
+    return None
 
 
 def transaction_status(transaction, txid):

app/economy/utils.py

@@ -112,3 +112,27 @@ def etherscan_link(txid):
 
     """
     return f'https://etherscan.io/tx/{txid}'
+
+
+def watch_txn(tx_id):
+    """Watches the Ethereum txn for updates (mainly being dropped/replaced)
+
+    Args:
+        txid (str): The transaction ID.
+
+    Returns:
+        str: The Etherscan TX URL.
+
+    """
+    import json
+    import requests
+    from django.conf import settings
+    args = {
+      "apiKey": settings.BLOCKNATIVE_API,
+      "hash": tx_id,
+      "blockchain": "ethereum",
+      "network": "main"
+    }
+    url = "https://api.blocknative.com/transaction"
+    response = requests.post(url, json=args)
+    print(response.text)

app/kudos/management/commands/update_tx_status.py

@@ -34,6 +34,7 @@ class Command(BaseCommand):
 
     help = 'gets the tx status of all SendCryptoAssets'
 
+    # processes grant contributions
     def process_grants_contribs(self):
         from grants.models import Contribution
         contributions = Contribution.objects.filter(tx_cleared=False)
@@ -51,7 +52,7 @@ def process_grants_contribs(self):
             contrib.update_tx_status()
             contrib.save()
 
-
+    # processes all crypto assets
     def process_acm(self):
         non_terminal_states = ['pending', 'na', 'unknown']
         for obj_type in all_sendcryptoasset_models():

app/perftools/views.py

@@ -1,5 +1,10 @@
+import json
+
+from django.conf import settings
+from django.http import HttpResponse
 from django.shortcuts import render
 from django.views.decorators.cache import cache_page
+from django.views.decorators.csrf import csrf_exempt
 
 from app.sitemaps import sitemaps
 
@@ -8,3 +13,14 @@
 def sitemap(request, section=None, template_name='sitemap.xml', mimetype='application/xml'): 
     from django.contrib.sitemaps.views import sitemap
     return sitemap(request, sitemaps, section, template_name, mimetype)
+
+
+@csrf_exempt
+def blocknative(request):
+    from economy.models import TXUpdate
+    body = json.loads(request.body)
+    if body['apiKey'] == settings.BLOCKNATIVE_API:
+        txu = TXUpdate.objects.create(body=body)
+        txu.process_callbacks()
+        txu.save()
+    return HttpResponse(status=200)