Snowplow Postgres Loader Error

Hi!

I’m trying to run the snowplow postgres loader but I’m getting the following error:

{
  "schema": "iglu:com.snowplowanalytics.snowplow.badrows/loader_iglu_error/jsonschema/2-0-0",
  "data": {
    "processor": { "artifact": "snowplow-postgres-loader", "version": "0.1.0" },
    "failure": [
      {
        "schemaCriterion": "iglu:com.snowplowanalytics.snowplow/web_page/jsonschema/1-*-*",
        "error": {
          "error": "ResolutionError",
          "lookupHistory": [
            {
              "repository": "Iglu Central",
              "errors": [{ "error": "NotFound" }],
              "attempts": 2,
              "lastAttempt": "2020-10-27T14:33:07.242Z"
            },
            {
              "repository": "Iglu Central - Mirror 01",
              "errors": [{ "error": "NotFound" }],
              "attempts": 1,
              "lastAttempt": "2020-10-27T14:33:06.748Z"
            },
            {
              "repository": "Iglu Client Embedded",
              "errors": [{ "error": "NotFound" }],
              "attempts": 1,
              "lastAttempt": "2020-10-27T14:33:06.749Z"
            }
          ]
        }
      }
    ],
    "payload": {
      "app_id": "website",
      "platform": "web",
      "etl_tstamp": "2020-10-26T14:33:06.312Z",
      "collector_tstamp": "2020-10-26T14:33:04.610Z",
      "dvce_created_tstamp": "2020-10-26T14:33:00.909Z",
      "event": "page_ping",
      "event_id": "4f87b4c2-87df-417d-85d2-412a6cf80016",
      "txn_id": null,
      "name_tracker": "cf",
      "v_tracker": "js-2.10.2",
      "v_collector": "ssc-0.15.0-kinesis",
      "v_etl": "stream-enrich-0.21.0-common-0.37.0",
      "user_id": null,
      "user_ipaddress": "xxxxxxxxxxx",
      "user_fingerprint": "129911700",
      "domain_userid": "16d83422-4de2-4ba7-9a5e-54e0e018d71e",
      "domain_sessionidx": 2,
      "network_userid": "88fba23f-9ee2-4631-8cb8-278dc33e327d",
      "geo_country": null,
      "geo_region": null,
      "geo_city": null,
      "geo_zipcode": null,
      "geo_latitude": null,
      "geo_longitude": null,
      "geo_region_name": null,
      "ip_isp": null,
      "ip_organization": null,
      "ip_domain": null,
      "ip_netspeed": null,
      "page_url": "https://www.raizs.com.br/ovos-organicos-grande--alimentos-delivery-sp/p?idsku=463&gclid=Cj0KCQjwxNT8BRD9ARIsAJ8S5xbyHgYlSZdZnQu5viMMs3BcOKgzRe6jSN3v_klEyHZBvx-55CYZnr4aAveVEALw_wcB",
      "page_title": "Ovos Orgânicos 10 unidades - Fazenda da Toca - Raizs",
      "page_referrer": null,
      "page_urlscheme": "https",
      "page_urlhost": "www.raizs.com.br",
      "page_urlport": 443,
      "page_urlpath": "/",
      "page_urlquery": "idsku=463&gclid=Cj0KCQjwxNT8BRD9ARIsAJ8S5xbyHgYlSZdZnQu5viMMs3BcOKgzRe6jSN3v_klEyHZBvx-55CYZnr4aAveVEALw_wcB",
      "page_urlfragment": null,
      "refr_urlscheme": null,
      "refr_urlhost": null,
      "refr_urlport": null,
      "refr_urlpath": null,
      "refr_urlquery": null,
      "refr_urlfragment": null,
      "refr_medium": null,
      "refr_source": null,
      "refr_term": null,
      "mkt_medium": null,
      "mkt_source": null,
      "mkt_term": null,
      "mkt_content": null,
      "mkt_campaign": null,
      "contexts": {
        "schema": "iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0",
        "data": [
          {
            "schema": "iglu:com.snowplowanalytics.snowplow/web_page/jsonschema/1-0-0",
            "data": { "id": "3b2cdd37-fa52-4d0d-a71f-941ce6772283" }
          }
        ]
      },
      "se_category": null,
      "se_action": null,
      "se_label": null,
      "se_property": null,
      "se_value": null,
      "unstruct_event": null,
      "tr_orderid": null,
      "tr_affiliation": null,
      "tr_total": null,
      "tr_tax": null,
      "tr_shipping": null,
      "tr_city": null,
      "tr_state": null,
      "tr_country": null,
      "ti_orderid": null,
      "ti_sku": null,
      "ti_name": null,
      "ti_category": null,
      "ti_price": null,
      "ti_quantity": null,
      "pp_xoffset_min": 0,
      "pp_xoffset_max": 0,
      "pp_yoffset_min": 428,
      "pp_yoffset_max": 771,
      "useragent": "Mozilla/5.0 (Linux; Android 8.0.0; SM-G570M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.110 Mobile Safari/537.36",
      "br_name": null,
      "br_family": null,
      "br_version": null,
      "br_type": null,
      "br_renderengine": null,
      "br_lang": "pt-BR",
      "br_features_pdf": null,
      "br_features_flash": null,
      "br_features_java": null,
      "br_features_director": null,
      "br_features_quicktime": null,
      "br_features_realplayer": null,
      "br_features_windowsmedia": null,
      "br_features_gears": null,
      "br_features_silverlight": null,
      "br_cookies": true,
      "br_colordepth": "24",
      "br_viewwidth": 360,
      "br_viewheight": 616,
      "os_name": null,
      "os_family": null,
      "os_manufacturer": null,
      "os_timezone": "America/Argentina/Buenos_Aires",
      "dvce_type": null,
      "dvce_ismobile": null,
      "dvce_screenwidth": 360,
      "dvce_screenheight": 640,
      "doc_charset": "UTF-8",
      "doc_width": 360,
      "doc_height": 4457,
      "tr_currency": null,
      "tr_total_base": null,
      "tr_tax_base": null,
      "tr_shipping_base": null,
      "ti_currency": null,
      "ti_price_base": null,
      "base_currency": null,
      "geo_timezone": null,
      "mkt_clickid": null,
      "mkt_network": null,
      "etl_tags": null,
      "dvce_sent_tstamp": "2020-10-26T14:33:00.933Z",
      "refr_domain_userid": null,
      "refr_dvce_tstamp": null,
      "derived_contexts": {},
      "domain_sessionid": "028a821f-1bf7-42f8-ad07-df02128607c1",
      "derived_tstamp": "2020-10-26T14:33:04.586Z",
      "event_vendor": "com.snowplowanalytics.snowplow",
      "event_name": "page_ping",
      "event_format": "jsonschema",
      "event_version": "1-0-0",
      "event_fingerprint": null,
      "true_tstamp": null
    }
  }
}

Does anyone knows what’s wrong?

resolver.json:

{
  "schema": "iglu:com.snowplowanalytics.iglu/resolver-config/jsonschema/1-0-2",
  "data": {
    "cacheSize": 500,
    "cacheTtl": 600,
    "repositories": [
      {
        "name": "Iglu Central",
        "priority": 1,
        "vendorPrefixes": ["com.snowplowanalytics"],
        "connection": {
          "http": {
            "uri": "http://iglucentral.com"
          }
        }
      },
      {
        "name": "Iglu Central - Mirror 01",
        "priority": 1,
        "vendorPrefixes": ["com.snowplowanalytics"],
        "connection": {
          "http": {
            "uri": "http://mirror01.iglucentral.com"
          }
        }
      }
    ]
  }
}

config.json

{
  "schema": "iglu:com.snowplowanalytics.snowplow.storage/postgresql_config/jsonschema/3-0-0",
  "data": {
    "name": "Acme Ltd. Snowplow Postgres",
    "id": "5c5e4353-4eeb-43da-98f8-2de6dc7fa947",
    "source": {
      "kinesis": {
        "appName": "xxxx",
        "streamName": "xxxx",
        "region": "xxxx",
        "initialPosition": "TRIM_HORIZON"
      }
    },
    "host": "xxxxx",
    "port": 5432,
    "database": "xxxxx",
    "username": "xxxx",
    "password": "xxxx",
    "schema": "atomic",
    "sslMode": "REQUIRE",
    "purpose": "ENRICHED_EVENTS"
  }
}

I think it’s got to be something to do with your network - you’re correctly pointing to Iglu Central (which is publicly available), but it’s returning not found. Iglu Central definitely isn’t down (we’d know about that kind of problem for sure!).

If I had to guess, the first things I’d look for are whether there’ either some IP blocklist or firewall in place somewhere that’s preventing the connection, or some broad rule for https-only.

I thought that too. But running wget http://iglucentral.com inside the container I got a 200 ok.
And I’m running in local machine with no network rules.

Hi @dehbmarques,

As it’s mentioned on Postgres Loader docs page, it is required that all schemas (not just custom ones) are hosted on Iglu Server 0.6.0+.

Your error indicates that it cannot find a mapping for iglu:com.snowplowanalytics.snowplow/web_page/jsonschema/1-*-* (not the actual schema) and these endpoints are available only on Iglu Servers.

Let us know how is it going with Postgres Loader.