\set ON_ERROR_STOP true
-------------------------------------------------------------------------------
-- Custom types: (global/public)
-------------------------------------------------------------------------------

DO $PUBLIC$
BEGIN
IF current_schema() = 'public' THEN

IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'view_type') THEN
CREATE TYPE VIEW_TYPE AS ENUM ('RBAC', 'FUNCTION');
END IF;

IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'change_operation') THEN
CREATE TYPE CHANGE_OPERATION AS ENUM ('ADD', 'CHANGE', 'REMOVE', 'UNTRACKED');
END IF;

IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'agent_exec_status') THEN
CREATE TYPE AGENT_EXEC_STATUS AS ENUM ('OK', 'FAIL');
END IF;

IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'patch_status') THEN
CREATE TYPE PATCH_STATUS AS ENUM ('INSTALLED', 'AVAILABLE');
END IF;

IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'last_seen_direction') THEN
CREATE TYPE LAST_SEEN_DIRECTION AS ENUM ('INCOMING', 'OUTGOING');
END IF;

IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'promise_outcome') THEN
CREATE TYPE PROMISE_OUTCOME AS ENUM ('KEPT', 'NOTKEPT', 'REPAIRED');
END IF;

-------------------------------------------------------------------------------
-- Meta tables to handle superhub partitioned table schema
-------------------------------------------------------------------------------

DROP TABLE IF EXISTS public.__Views; -- want to start fresh each time
CREATE TABLE IF NOT EXISTS public.__Views (
       ViewType            VIEW_TYPE,
       SourceName          TEXT   PRIMARY KEY,
       ViewName            TEXT   NOT NULL,
       HostKeyColumn       TEXT,
       Id                  BIGSERIAL -- use this for ordering the views the same as in this sql file
       );
COMMENT ON TABLE public.__Views IS 'A list of table or function views to apply.';

-------------------------------------------------------------------------------
-- Functions and Operators: (global/public)
-------------------------------------------------------------------------------

CREATE OR REPLACE FUNCTION reverse_lower_eq(text,text)
RETURNS BOOL AS
    'select lower($2) = lower($1)'
LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;

-------------------------------------------------------------------------------

DROP OPERATOR IF EXISTS ~~~% (text, text);
CREATE OPERATOR ~~~% (procedure = reverse_lower_eq, leftarg = text, rightarg = text);

-------------------------------------------------------------------------------

CREATE OR REPLACE FUNCTION reverse_lower_like(text,text)
RETURNS BOOL AS
    'select lower($2) LIKE lower($1)'
LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;

-------------------------------------------------------------------------------

DROP OPERATOR IF EXISTS ~~~%~ (text, text);
CREATE OPERATOR ~~~%~ (procedure = reverse_lower_like, leftarg = text, rightarg = text);

-------------------------------------------------------------------------------

CREATE OR REPLACE FUNCTION verify_table(in_table_name TEXT, in_table_schema TEXT DEFAULT current_schema()) RETURNS BOOL AS $$
DECLARE rec RECORD;
BEGIN
  SELECT * FROM information_schema.tables WHERE table_name = LOWER(in_table_name) AND table_schema = in_table_schema AND table_type = 'BASE TABLE' INTO rec;
  RAISE DEBUG 'verify_table(%,%) found information_schema.tables row: %', in_table_name, in_table_schema, rec;
  RETURN FOUND;
END;
$$ LANGUAGE plpgsql;

-------------------------------------------------------------------------------

CREATE OR REPLACE FUNCTION verify_schema(in_schema_name TEXT) RETURNS BOOL AS $$
DECLARE rec RECORD;
BEGIN
  SELECT * FROM information_schema.schemata WHERE schema_name = in_schema_name INTO rec;
  RAISE DEBUG 'verify_schema(%) found information_schema.schemata row: %', in_schema_name, rec;
  RETURN FOUND;
END;
$$ LANGUAGE plpgsql;


CREATE OR REPLACE FUNCTION verify_materialized_view(in_matview_name TEXT, in_schema_name TEXT DEFAULT current_schema()) RETURNS BOOL AS $$
DECLARE rec RECORD;
BEGIN
  SELECT * from pg_matviews WHERE schemaname = in_schema_name AND matviewname = in_matview_name INTO rec;
  RAISE DEBUG 'verify_materialized_view(%,%) found pg_matviews row: %', in_matview_name, in_schema_name, rec;
  RETURN FOUND;
END;
$$ LANGUAGE plpgsql;

CREATE OR REPLACE FUNCTION verify_function(in_function_name TEXT, in_nargs INT DEFAULT 0, in_schema_name TEXT DEFAULT current_schema()) RETURNS BOOL AS $$
DECLARE rec RECORD;
BEGIN
  SELECT * FROM pg_proc WHERE proname = in_function_name
    AND prokind = 'f'
    AND pronargs = in_nargs
    AND pronamespace =
      (SELECT oid
         FROM pg_namespace
        WHERE nspname = in_schema_name) INTO rec;
  RAISE DEBUG 'verify_function(%,%,%) found pg_proc row: %', in_function_name,  in_nargs, in_schema_name, rec;
  RETURN FOUND;
END;
$$ LANGUAGE plpgsql;

-------------------------------------------------------------------------------

CREATE OR REPLACE FUNCTION get_hub_id(hostkey_search TEXT) RETURNS NUMERIC AS $$
DECLARE out_hub_id NUMERIC;
BEGIN
  SELECT hub_id FROM __hubs where hostkey = hostkey_search INTO out_hub_id;
  IF out_hub_id IS NULL THEN
    RAISE EXCEPTION 'No hostkey of % in __hubs', hostkey_search;
  END IF;
  RAISE DEBUG 'get_hub_id(%), got hub_id=%', hostkey_search, out_hub_id;
  RETURN out_hub_id;
END;
$$ LANGUAGE plpgsql;

-------------------------------------------------------------------------------

-- table name parameters MAY include a namespace portion like <namespace/schema>.<table_name>
-- if no namespace is included in the table name then the current_schema() will be used.
CREATE OR REPLACE FUNCTION verify_inheritance(parent_table TEXT, child_table TEXT) RETURNS BOOL AS $$
DECLARE value BOOLEAN;
        parent_info TEXT[];
        child_info TEXT[];
        parent_table_name TEXT;
        parent_namespace TEXT;
        child_table_name TEXT;
        child_namespace TEXT;
        rec RECORD;

BEGIN
  parent_info = regexp_split_to_array(parent_table, '\.');
  IF parent_info[2] IS NULL THEN
    parent_table_name = parent_info[1];
    parent_namespace = current_schema();
  ELSE
    parent_table_name = parent_info[2];
    parent_namespace = parent_info[1];
  END IF;

  child_info = regexp_split_to_array(child_table, '\.');
  IF child_info[2] IS NULL THEN
    child_table_name = child_info[1];
    child_namespace = current_schema();
  ELSE
    child_table_name = child_info[2];
    child_namespace = child_info[1];
  END IF;

  SELECT EXISTS
    (SELECT *
     FROM pg_inherits
     WHERE inhparent =
       (SELECT oid
        FROM pg_class
        WHERE relname = parent_table_name
          AND relnamespace =
           (SELECT oid
            FROM pg_namespace
            WHERE nspname = parent_namespace))
       AND inhrelid =
         (SELECT oid
          FROM pg_class
          WHERE relname = child_table_name
            AND relnamespace =
             (SELECT oid
              FROM pg_namespace
              WHERE nspname = child_namespace))) INTO value;
  RETURN value;
END;
$$ LANGUAGE plpgsql;

-------------------------------------------------------------------------------
-- Functions for cfdb database used by Mission Portal (global/public)
-------------------------------------------------------------------------------


-- remove '{ ',' '} and {" "," "} from inventory slist fields
CREATE OR REPLACE FUNCTION cf_clearSlist(varchar) RETURNS varchar AS
$$ 	SELECT trim(regexp_replace(regexp_replace($1,'\'',\''|\'', \''|", ?"', ', ', 'g'), '(^\s*{\''|\''}|{"|"}\s*$)', '', 'g'))
$$ LANGUAGE SQL IMMUTABLE RETURNS NULL on NULL INPUT;

-------------------------------------------------------------------------------

-- convert string to number, return NULL if not able to convert
-- Using EXCEPTION creates subtransaction that is not PARALLEL SAFE
CREATE OR REPLACE FUNCTION cf_convertToNum(text) RETURNS numeric AS $$
    DECLARE x NUMERIC;
    BEGIN
        x = $1::NUMERIC;
        RETURN x;
    EXCEPTION WHEN others THEN
        RETURN NULL;
    END;
$$ LANGUAGE plpgsql IMMUTABLE RETURNS NULL on NULL INPUT;

-------------------------------------------------------------------------------

CREATE OR REPLACE FUNCTION use_default_hub_schema() RETURNS VOID AS $$
BEGIN
  IF public.is_superhub() THEN
    RAISE DEBUG 'In use_default_hub_schema(), switching to hub_0 as we are on superhub';
    SET SCHEMA 'hub_0';
  END IF;
END;
$$ LANGUAGE plpgsql;

-- this is here so it appears just before it's first use
CREATE OR REPLACE FUNCTION is_superhub() RETURNS BOOL AS $$
BEGIN
  -- __hubs presence is our indicator of superhub or not
  RETURN public.verify_table('__hubs','public');
END;
$$ LANGUAGE plpgsql;

-------------------------------------------------------------------------------

-- return hostkeys and grouped attributes values as table by attribute name
CREATE OR REPLACE FUNCTION getHostkeyAndValuesForAttribute(attrSubStr TEXT) RETURNS TABLE (
   hostkey   text,
   inventoryValue text
  ) AS
$func$

DECLARE sqlStr TEXT;

BEGIN

sqlStr := $x$
   SELECT HostKey,
	  string_agg(level2.value, ', ' ORDER BY lower(level2.value) ASC )
	  FROM
		   (SELECT
			inventory.HostKey,
			inventory.value
		   FROM
		      inventory
		   WHERE
			 $x$|| quote_nullable(attrSubStr)  ||$x$  = ANY (inventory.metatags)
		   GROUP BY
			inventory.hostkey,
			inventory.value) as level2
	GROUP BY HostKey;
    $x$;
RETURN query
EXECUTE sqlStr
USING   attrSubStr;

  end;  $func$ language plpgsql PARALLEL SAFE;

-------------------------------------------------------------------------------

-- return hostkeys and grouped attributes values as table by key name
CREATE OR REPLACE FUNCTION getHostkeyAndValuesForKeyName(keyName TEXT) RETURNS TABLE (
   hostkey   text,
   inventoryValue text
  ) AS
$func$

DECLARE sqlStr TEXT;

BEGIN

sqlStr := $x$
   SELECT HostKey,
	  string_agg(level2.value, ', ' ORDER BY lower(level2.value) ASC )
	  FROM
		   (SELECT
			inventory.HostKey,
			inventory.value
		   FROM
		      inventory
		   WHERE
			 keyname = $x$  || quote_nullable(keyName) || $x$
		   GROUP BY
			inventory.hostkey,
			inventory.value) as level2
	GROUP BY HostKey;
    $x$;
RETURN query
EXECUTE sqlStr
USING   keyName;

  end;  $func$ language plpgsql PARALLEL SAFE;

-------------------------------------------------------------------------------

-- generate partition name for promiselog
CREATE OR REPLACE FUNCTION promise_log_partition_name_create(date TIMESTAMPTZ, outcome PROMISE_OUTCOME)
RETURNS TEXT  AS
$BODY$
DECLARE
    name TEXT;

BEGIN
    name := '__promiselog_' || outcome || '_' || to_char(date, 'YYYY-MM-DD');
    RETURN name;
END;
$BODY$
LANGUAGE plpgsql;

-------------------------------------------------------------------------------

-- Auto paritioning function for promise_log.
-- Not generic at this point as it creates indexing as well
CREATE OR REPLACE FUNCTION promise_log_partition_function()
RETURNS TRIGGER AS
$BODY$
DECLARE
    _tablename text;
    _startdate text;
    _enddate text;
    _outcome public.PROMISE_OUTCOME;

BEGIN
    -- Generate partition name based on ChangeTimeStamp and PromiseOutcome
    _outcome := NEW."promiseoutcome";
    _tablename := public.promise_log_partition_name_create(NEW."changetimestamp"::TIMESTAMPTZ, _outcome::public.PROMISE_OUTCOME);

    -- Insert the current record into the correct partition, which we are sure will now exist.
    EXECUTE 'INSERT INTO ' || TG_TABLE_SCHEMA || '.' || quote_ident(_tablename) || ' VALUES ($1.*)' USING NEW;
    RETURN NULL;

    EXCEPTION
    WHEN undefined_table THEN

    PERFORM public.promise_log_partition_create(NEW."changetimestamp", 1, _outcome, TG_TABLE_SCHEMA);
    EXECUTE 'INSERT INTO ' || TG_TABLE_SCHEMA || '.' || quote_ident(_tablename) || ' VALUES ($1.*)' USING NEW;

    RETURN NULL;
END;
$BODY$
LANGUAGE plpgsql;

-------------------------------------------------------------------------------

DROP FUNCTION IF EXISTS promise_log_partition_create(TIMESTAMPTZ, INTEGER, PROMISE_OUTCOME);
DROP FUNCTION IF EXISTS promise_log_partition_create(TIMESTAMPTZ, INTEGER, PROMISE_OUTCOME, TEXT);
CREATE OR REPLACE FUNCTION promise_log_partition_create(now TIMESTAMPTZ, partitions INTEGER, outcome PROMISE_OUTCOME, _schema TEXT DEFAULT current_schema())
RETURNS VOID AS
$BODY$

DECLARE
    _interval INTERVAL;
    _now TIMESTAMPTZ;
    _table_time TIMESTAMPTZ;
    _tablename TEXT;

BEGIN
    _interval := quote_literal(partitions - 1|| ' day');
    _now := to_timestamp(to_char(now, 'YYYY-MM-DD'), 'YYYY-MM-DD');

    FOR _table_time IN
        SELECT ts::timestamp
            FROM generate_series(_now::timestamp, _now::timestamp + _interval, '1 day'::INTERVAL) as ts
    LOOP
        _tablename := public.promise_log_partition_name_create(_table_time, outcome);

        IF NOT EXISTS (
            SELECT 1
            FROM   information_schema.tables
            WHERE  table_name = _tablename
            AND table_schema = _schema )
        THEN
            -- create new partition
            EXECUTE 'CREATE TABLE ' || quote_ident(_schema) || '.' || quote_ident(_tablename) || ' (
                CHECK ( "changetimestamp" >= ' || quote_literal(_table_time) || '::timestamptz
                AND "changetimestamp" < ' || quote_literal(_table_time + INTERVAL '1 day') || '::timestamptz
                AND "promiseoutcome" = ' || quote_literal(outcome) || '::public.PROMISE_OUTCOME )
                ) INHERITS (' || quote_ident(_schema) || '.__PromiseLog)';

            -- Indexes are defined per child, so we assign a default index that uses the partition columns
            EXECUTE 'CREATE INDEX IF NOT EXISTS ' || quote_ident(_tablename||'_hostkey') || ' ON ' || quote_ident(_tablename) || ' (HostKey) WITH (FILLFACTOR = 90)';
            PERFORM public.ensure_unique_index(quote_ident(_tablename||'_id'), quote_ident(_tablename), 'Id', 90);
            EXECUTE 'CREATE INDEX IF NOT EXISTS ' || quote_ident(_tablename||'_outcome') || ' ON ' || quote_ident(_tablename) || ' (PromiseOutcome) WITH (FILLFACTOR = 90)';
            EXECUTE 'CREATE INDEX IF NOT EXISTS ' || quote_ident(_tablename||'_changetimestamp') || ' ON ' || quote_ident(_tablename) || ' (ChangeTimeStamp) WITH (FILLFACTOR = 90)';
            EXECUTE 'CREATE INDEX IF NOT EXISTS ' || quote_ident(_tablename||'_policyfile') || ' ON ' || quote_ident(_tablename) || ' (lower(PolicyFile)) WITH (FILLFACTOR = 90)';
            EXECUTE 'CREATE INDEX IF NOT EXISTS ' || quote_ident(_tablename||'_promisehash') || ' ON ' || quote_ident(_tablename) || ' (lower(PromiseHash)) WITH (FILLFACTOR = 90)';
            EXECUTE 'CREATE INDEX IF NOT EXISTS ' || quote_ident(_tablename||'_bundlename') || ' ON ' || quote_ident(_tablename) || ' (lower(BundleName)) WITH (FILLFACTOR = 90)';
            EXECUTE 'CREATE INDEX IF NOT EXISTS ' || quote_ident(_tablename||'_promisetype') || ' ON ' || quote_ident(_tablename) || ' (lower(PromiseType)) WITH (FILLFACTOR = 90)';
            EXECUTE 'CREATE INDEX IF NOT EXISTS ' || quote_ident(_tablename||'_promiser') || ' ON ' || quote_ident(_tablename) || ' (lower(Promiser)) WITH (FILLFACTOR = 90)';
            EXECUTE 'CREATE INDEX IF NOT EXISTS ' || quote_ident(_tablename||'_stackpath') || ' ON ' || quote_ident(_tablename) || ' (lower(StackPath)) WITH (FILLFACTOR = 90)';
            EXECUTE 'CREATE INDEX IF NOT EXISTS ' || quote_ident(_tablename||'_promisehandle') || ' ON ' || quote_ident(_tablename) || ' (lower(PromiseHandle)) WITH (FILLFACTOR = 90)';

        END IF;
    END LOOP;

    RETURN;
END;
$BODY$
LANGUAGE plpgsql;

-------------------------------------------------------------------------------

-- Cleanup promiselog based on interval
CREATE OR REPLACE FUNCTION promise_log_partition_cleanup(outcome PROMISE_OUTCOME, keep_days INTERVAL)
RETURNS VOID AS
$BODY$
DECLARE
    _tablename text;

BEGIN
    FOR _tablename IN
        SELECT table_name
        FROM (
            SELECT table_name, to_timestamp(substring(table_name from (15 + char_length(outcome::text)) for 10), 'YYYY MM DD') as timestamp
            FROM information_schema.tables
            WHERE table_schema='public'
                AND table_type='BASE TABLE'
                AND table_name ILIKE '__promiselog_' || outcome || '_%'
        ) as sub
        WHERE timestamp < NOW() - keep_days
    LOOP
        EXECUTE 'DROP TABLE ' || quote_ident(_tablename);
    END LOOP;

    RETURN;
END;
$BODY$
LANGUAGE plpgsql;

-------------------------------------------------------------------------------

-- In postgresql 12.1 `DROP FUNCTION IF EXISTS` was changed to give
-- an ERROR if there is a matching procedure of the same name. Check and drop manually.
DO $$
BEGIN
  IF verify_function('update_variables_dictionary') THEN
    DROP FUNCTION IF EXISTS update_variables_dictionary();
  END IF;
END$$;

-- Create procedure to add new custom variables to variables_dictionary
DROP PROCEDURE IF EXISTS update_variables_dictionary();
CREATE PROCEDURE update_variables_dictionary()
  LANGUAGE plpgsql
  AS $$
BEGIN
  IF pg_is_in_recovery() THEN
       RAISE LOG 'PostgreSQL is in recovery, skipping update of variables dictionary';
  ELSE
    INSERT INTO variables_dictionary ("attribute_name", "category","type","readonly","keyname")
        SELECT distinct on (attribute_name) "substring"(__variables.metatags::text, 'attribute_name=["]?([^"},]+)'::text) AS attribute_name,
        'User defined'::text as category,
        variabletype as type,
        '0'::int as readonly,
        comp as keyname
        FROM __variables
        WHERE  ('inventory'::text = ANY (__variables.metatags)) AND NOT ('attribute_name=none'::text = ANY (__variables.metatags))
        AND "substring"(__variables.metatags::text, 'attribute_name=["]?([^"},]+)'::text) NOT IN (SELECT attribute_name FROM variables_dictionary )
        GROUP BY __variables.metatags , variabletype, comp;

    INSERT INTO variables_dictionary ("attribute_name", "category","type","readonly","keyname")
        SELECT distinct on (attribute_name) "substring"(__contexts.metatags::text, 'attribute_name=["]?([^"},]+)'::text) AS attribute_name,
        'User defined'::text as category,
        'string'::text as type,
        '0'::int as readonly,
        ''::text as keyname
        FROM __contexts
        WHERE  ('inventory'::text = ANY (__contexts.metatags)) AND NOT ('attribute_name=none'::text = ANY (__contexts.metatags))
        AND "substring"(__contexts.metatags::text, 'attribute_name=["]?([^"},]+)'::text) NOT IN (SELECT attribute_name FROM variables_dictionary )
        GROUP BY __contexts.metatags;
  END IF;
END;
$$;
COMMENT ON PROCEDURE update_variables_dictionary IS 'Update variables_dictionary table';

-------------------------------------------------------------------------------

-- In postgresql 12.1 `DROP FUNCTION IF EXISTS` was changed to give
-- an ERROR if there is a matching procedure of the same name. Check and drop manually.
DO $$
BEGIN
  IF verify_function('disable_unreported_variables') THEN
    DROP FUNCTION IF EXISTS disable_unreported_variables();
  END IF;
END$$;

-- Disable variable which don't have any reports data
DROP PROCEDURE IF EXISTS disable_unreported_variables();
CREATE PROCEDURE disable_unreported_variables()
  LANGUAGE plpgsql
  AS $$
BEGIN
  IF pg_is_in_recovery() THEN
    RAISE LOG 'PostgreSQL is in recovery, skipping disabling unreported variables';
  ELSE
    UPDATE variables_dictionary SET enabled = 0;
    UPDATE variables_dictionary SET enabled = 1 WHERE attribute_name IN ( SELECT distinct on (attribute_name) "substring"(__variables.metatags::text, 'attribute_name=["]?([^"},]+)'::text) AS attribute_name  FROM __variables) OR attribute_name IN ( SELECT distinct on (attribute_name) "substring"(__contexts.metatags::text, 'attribute_name=["]?([^"},]+)'::text) AS attribute_name  FROM __contexts);
  END IF;
END;
$$;
COMMENT ON PROCEDURE disable_unreported_variables IS 'Disable unreported variables in variables_dictionary table';

-------------------------------------------------------------------------------

CREATE OR REPLACE FUNCTION public.get_rbac_hostkeys()
RETURNS TABLE (hostkey text) AS $body$
BEGIN
    RETURN QUERY SELECT ContextCache.hostkey FROM public.ContextCache WHERE ContextVector @@ to_tsquery('simple', current_setting('rbac.filter'));
END;
$body$ LANGUAGE plpgsql;

COMMENT ON FUNCTION get_rbac_hostkeys() IS 'Get hostkeys of hosts filtered by the current rbac.filter value';

-------------------------------------------------------------------------------

CREATE OR REPLACE FUNCTION public.ensure_view(view_type public.VIEW_TYPE, source_name TEXT, view_name TEXT default '', hostkey_column TEXT default 'HostKey') RETURNS VOID AS $body$
BEGIN
  IF current_schema() != 'public' THEN
    RETURN;
  END IF;
  -- special general rule for our schema, private tables are prefixed with '__' and have
  -- rbac views on top without the prefix.
  EXECUTE 'INSERT INTO public.__Views VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING' USING view_type, source_name, view_name, hostkey_column;

  IF view_type = 'RBAC' THEN
    IF view_name = '' THEN
      view_name := replace(source_name, '__', '');
    END IF;

    EXECUTE format('
CREATE OR REPLACE VIEW %s WITH (security_barrier) AS
  SELECT p.* FROM %s AS p
  WHERE  (
    CASE WHEN
      coalesce(current_setting(''rbac.filter'', true), '''') = ''''
    THEN
      p.%s IS NOT NULL
    ELSE
      p.%s IN (SELECT * FROM get_rbac_hostkeys())
    END )',
       view_name,
       source_name,
       hostkey_column,
       hostkey_column);
  ELSIF view_type = 'FUNCTION' THEN
    EXECUTE format('SELECT %s();', source_name);
  END IF;
END;
$body$ LANGUAGE plpgsql;

-------------------------------------------------------------------------------

CREATE OR REPLACE FUNCTION ensure_primary_key(
    table_name TEXT,
    key_name TEXT,
    keys TEXT) RETURNS void AS
$BODY$
DECLARE
  key_text TEXT;
  target_schema TEXT default 'public';
BEGIN
  key_text := keys;

  IF public.is_superhub() THEN
    key_text := key_text || ', hub_id';
    target_schema := 'hub_0';
  END IF;

  IF NOT EXISTS (SELECT constraint_name
                   FROM information_schema.table_constraints
                  WHERE constraint_name = key_name
                    AND table_schema = target_schema) THEN
    EXECUTE format('ALTER TABLE %s.%s
                      ADD CONSTRAINT %s PRIMARY KEY (%s);',
                   target_schema,key_name,
                   key_name,key_text);
  END IF;
END;
$BODY$ LANGUAGE plpgsql STRICT;

COMMENT ON FUNCTION ensure_primary_key (text, text, text) IS 'create primary key in proper schema if not already present';

CREATE OR REPLACE FUNCTION ensure_unique_index(
    index_name TEXT,
    table_name TEXT,
    keys TEXT default 'hostkey', -- comma separated keys for index
    fill_factor INTEGER default 90) RETURNS void AS
$BODY$
BEGIN
    PERFORM public.ensure_index(index_name, table_name, keys, fill_factor, 'UNIQUE');
END;
$BODY$ LANGUAGE plpgsql STRICT;

COMMENT ON FUNCTION ensure_unique_index (text, text, text, integer) IS 'create unique index in proper schema';

CREATE OR REPLACE FUNCTION ensure_index(
    index_name TEXT,
    table_name TEXT,
    keys TEXT default 'hostkey', -- comma separated keys for index
    fill_factor INTEGER default 90,
    unique_text TEXT default 'UNIQUE') RETURNS void AS
$BODY$
DECLARE
  key_text TEXT;
  constraint_name TEXT;
BEGIN
  key_text := keys;
  IF public.is_superhub() THEN
    key_text := key_text || ', hub_id';
  END IF;

  -- ENT-5268: If unique_text is UNIQUE and not a materialized view we can create instead
  -- a CONSTRAINT, which automatically creates an associated UNIQUE INDEX.
  -- This allows clients such as cf-hub to call use_default_hub_schema() to change
  -- to hub_0 (primary hub schema for superhub enabled hubs) and operate "as normal".
  IF unique_text = 'UNIQUE'
    AND NOT public.verify_materialized_view(table_name)
    AND NOT public.is_superhub() THEN
    RAISE DEBUG 'using constraint instead of just unique index on %', table_name;
    constraint_name := index_name;
    index_name := index_name || '_index';
  END IF;

  EXECUTE format('
CREATE %s INDEX IF NOT EXISTS %s ON %s (%s) WITH (FILLFACTOR = %s);',
    unique_text,
    quote_ident(index_name),
    table_name,
    key_text,
    fill_factor);

  IF constraint_name IS NOT NULL THEN
    EXECUTE format('
ALTER TABLE %s DROP CONSTRAINT IF EXISTS %s;
DROP INDEX IF EXISTS %s;
ALTER TABLE %s ADD CONSTRAINT %s UNIQUE USING INDEX %s;',
      table_name,
      quote_ident(constraint_name),
      quote_ident(constraint_name),
      table_name,
      quote_ident(constraint_name),
      quote_ident(index_name));
  END IF;
END;
$BODY$ LANGUAGE plpgsql STRICT;

COMMENT ON FUNCTION ensure_index (text, text, text, integer, text) IS 'create index in proper schema';

-------------------------------------------------------------------------------

-- Create function  clean_historical_data
CREATE OR REPLACE FUNCTION cleanup_historical_data(
    table_name TEXT,
    order_by_column TEXT,
    items_number_to_keep INTEGER,
    hostkey_column TEXT default 'HostKey'
    ) RETURNS void AS
$BODY$
DECLARE
    row record;
BEGIN
    table_name := $1;
    order_by_column := $2;
    items_number_to_keep := $3;
    hostkey_column := $4;

    FOR row IN
        EXECUTE 'SELECT DISTINCT '|| hostkey_column ||' FROM '|| table_name ||''
    LOOP
        EXECUTE 'DELETE FROM '|| table_name ||' WHERE '|| order_by_column ||' NOT IN (SELECT '|| order_by_column ||' FROM '|| table_name ||' WHERE '|| hostkey_column ||' = '''|| row.host ||''' ORDER BY '|| order_by_column ||' DESC OFFSET 0 LIMIT '|| items_number_to_keep ||') AND '|| hostkey_column ||' = '''|| row.host ||'''';
    END LOOP;

  RETURN;
END;
$BODY$ LANGUAGE plpgsql STRICT;

COMMENT ON FUNCTION cleanup_historical_data (text, text, integer, text) IS 'Clean up stale records from table. Arguments: table name, order by column, number items to preserve, host key column name.';

END IF;
END $PUBLIC$;

-------------------------------------------------------------------------------
-- RBAC (beginning of most of the actual table DDL)
-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS ContextCache (
        HostKey             TEXT        NOT NULL,
        ContextVector       TSVECTOR    NOT NULL
    ) WITH (FILLFACTOR = 90);

SELECT public.ensure_unique_index('contextcache_uniq', 'contextcache');
CREATE INDEX IF NOT EXISTS context_cache_vector ON ContextCache USING gin(ContextVector);
COMMENT ON TABLE ContextCache IS 'Classes reported by each host presented as a vector for full text search. Maintenance of the table is shared between triggers and cf-hub';

-------------------------------------------------------------------------------
-- Current state tables:
-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __Hosts (
        HostKey                 TEXT                        NOT NULL,
        IsCallCollected         boolean                     DEFAULT false,
        LastReportTimeStamp     timestamp with time zone    DEFAULT NULL,
--      HostName                TEXT                        DEFAULT NULL,
        FirstReportTimeStamp    timestamp with time zone    DEFAULT CURRENT_TIMESTAMP,
        HostKeyCollisions       INTEGER                     DEFAULT 0,
        Deleted                 timestamp with time zone    DEFAULT NULL,
        IPAddress               TEXT                        DEFAULT NULL,
        CONSTRAINT hostkey_not_empty_check CHECK(hostkey <> '')
    ) WITH (FILLFACTOR = 70);

DO $PUBLIC$
BEGIN
IF current_schema() = 'public' THEN
ALTER TABLE __Hosts ADD COLUMN IF NOT EXISTS HostKeyCollisions INTEGER DEFAULT 0;
ALTER TABLE __Hosts ADD COLUMN IF NOT EXISTS Deleted timestamp with time zone DEFAULT NULL;
ALTER TABLE __Hosts ADD COLUMN IF NOT EXISTS IPAddress TEXT DEFAULT NULL;
ALTER TABLE __Hosts ALTER COLUMN LastReportTimeStamp SET DEFAULT NULL;
DO $$
  BEGIN
    ALTER TABLE "__hosts" DROP CONSTRAINT IF EXISTS hostkey_not_empty_check;
    ALTER TABLE "__hosts" ADD CONSTRAINT hostkey_not_empty_check CHECK(hostkey <> '');
    EXCEPTION
      WHEN check_violation THEN
        raise NOTICE 'Failed to add constraint hostkey_not_empty_check to __hosts table, there are entries that have empty hostkey. Please contact support@northern.tech';
  END;
$$
language plpgsql;

END IF;
END $PUBLIC$;

COMMENT ON TABLE __Hosts IS 'Internal table holding basic host information.';
COMMENT ON COLUMN __Hosts.HostKey IS 'Public key digest representing host identity (unique).';
COMMENT ON COLUMN __Hosts.IsCallCollected IS 'True when report collection has been initiated by the client at least one time.';
COMMENT ON COLUMN __Hosts.LastReportTimeStamp IS 'Timestamp of last successful report collection by cf-hub (seconds precision).';
COMMENT ON COLUMN __Hosts.FirstReportTimeStamp IS 'Timestamp of when the host first appeared in hosts table, when it was discovered for the first time by cf-hub.';
COMMENT ON COLUMN __Hosts.HostKeyCollisions IS 'Number of times cf-hub observed cookie mismatch (from protocol version 3)';
COMMENT ON COLUMN __Hosts.Deleted IS 'Timestamp when host was deleted via API call. When set, cf-hub will not collect reports from the host.';
COMMENT ON COLUMN __Hosts.IPAddress IS 'IP address that cf-hub last received reports from.';

DO $PUBLIC$
BEGIN
  IF current_schema() = 'public' THEN

GRANT ALL ON TABLE public.__hosts TO cfapache;

DROP PROCEDURE IF EXISTS upsert_host_with_ip(hostkey TEXT, ipaddress TEXT);
CREATE PROCEDURE upsert_host_with_ip(hostkey TEXT, ipaddress TEXT)
  LANGUAGE plpgsql
  AS $$
BEGIN
  IF public.is_superhub() THEN
    EXECUTE '
      INSERT INTO __hosts (hostkey, ipaddress)
      VALUES ($1, $2)
      ON CONFLICT (hostkey, hub_id)
      DO  UPDATE SET ipaddress = excluded.ipaddress'
      USING hostkey, ipaddress;
  ELSE
    EXECUTE '
      INSERT INTO __hosts (hostkey, ipaddress)
      VALUES ($1, $2)
      ON CONFLICT (hostkey)
      DO  UPDATE SET ipaddress = excluded.ipaddress'
      USING hostkey, ipaddress;
  END IF;
END
$$;

END IF; -- if current_schema() = 'public'
END $PUBLIC$;

SELECT public.ensure_unique_index('__hosts_uniq', '__hosts', 'hostkey', 70);

SELECT public.ensure_view('RBAC', '__Hosts', 'v_Hosts');

DO $PUBLIC$
BEGIN
  IF current_schema() = 'public' THEN

CREATE OR REPLACE FUNCTION ensure_hosts_not_reported_view() RETURNS VOID AS $body$
BEGIN
  CREATE OR REPLACE VIEW hosts_not_reported WITH (security_barrier) AS
    SELECT p.*
    FROM __hosts p
    WHERE (EXISTS (
            SELECT HostKey
            FROM ContextCache
            WHERE ContextVector @@ to_tsquery('simple', 'host.not.reported')
                AND p.hostkey = HostKey
        ) OR NOT EXISTS (
            SELECT HostKey
            FROM ContextCache
            WHERE p.hostkey = HostKey
        ))
        AND (firstreporttimestamp + INTERVAL '1 hour')::timestamptz < now()
        AND p.deleted IS NULL;
END;
$body$ LANGUAGE plpgsql;

PERFORM public.ensure_view('FUNCTION', 'ensure_hosts_not_reported_view');

COMMENT ON VIEW hosts_not_reported IS 'View to select never collected hosts';

END IF;
END $PUBLIC$;

-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __AgentStatus (
        HostKey                     		TEXT                          NOT NULL,
        AgentExecutionInterval      		INTEGER,
        LastAgentLocalExecutionTimeStamp 	timestamp with time zone,
        LastAgentExecutionStatus    		public.AGENT_EXEC_STATUS
    ) WITH (FILLFACTOR = 70);
COMMENT ON TABLE __AgentStatus IS 'High level summary of status of reporting host.';
COMMENT ON COLUMN __AgentStatus.AgentExecutionInterval IS 'The average interval between executions of cf-agent as reported by the client.';
COMMENT ON COLUMN __AgentStatus.LastAgentLocalExecutionTimeStamp IS 'Time of the last reported execution of cf-agent.';
COMMENT ON COLUMN __AgentStatus.LastAgentExecutionStatus IS 'Indication if the last reported agent execution was successful or had abnormal termination. (OK|FAIL)';

SELECT public.ensure_unique_index('__agentstatus_uniq', '__agentstatus', 'hostkey', 70);
SELECT public.ensure_view('RBAC', '__AgentStatus');

-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __Contexts (
        HostKey             TEXT                        NOT NULL,
        ContextName         TEXT                        NOT NULL,
        MetaTags            TEXT[],
        ChangeTimeStamp     timestamp with time zone
    ) WITH (FILLFACTOR = 90);

COMMENT ON TABLE __Contexts is 'Classes defined during the last reported execution of cf-agent.';
COMMENT ON COLUMN __Contexts.HostKey IS 'Public key digest representing host identity.';
COMMENT ON COLUMN __Contexts.ContextName IS 'Name of class that was defined.';
COMMENT ON COLUMN __Contexts.MetaTags IS 'MetaTags associated with the class.';
COMMENT ON COLUMN __Contexts.ChangeTimeStamp IS 'Timestamp from the perspective of the related host when the class is set in its current form. Note: If any of the context attributes change, the timestamp will be updated.';

SELECT public.ensure_unique_index('__contexts_uniq', '__contexts', 'hostkey, contextname', 90);
CREATE INDEX IF NOT EXISTS contexts_pattern_ops ON __Contexts (ContextName varchar_pattern_ops) WITH (FILLFACTOR = 90);
CREATE INDEX IF NOT EXISTS contexts_meta ON __Contexts (MetaTags) WITH (FILLFACTOR = 90);

SELECT public.ensure_view('RBAC', '__Contexts');

-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __Variables (
        HostKey             TEXT                        NOT NULL,
        NameSpace           TEXT                        DEFAULT NULL,
        Bundle              TEXT                        NOT NULL,
        VariableName        TEXT                        NOT NULL,
        VariableValue       TEXT                        DEFAULT NULL,
        VariableType        TEXT                        NOT NULL,
        comp                TEXT                        NOT NULL,
        MetaTags            TEXT[],
        ChangeTimeStamp     timestamp with time zone
    ) WITH (FILLFACTOR = 70);

SELECT public.ensure_unique_index('__variables_uniq', '__variables', 'hostkey, comp', 70);
CREATE INDEX IF NOT EXISTS variables_meta ON __Variables (MetaTags) WITH (FILLFACTOR = 70);
CREATE INDEX IF NOT EXISTS variables_variablename ON __Variables (VariableName) WITH (FILLFACTOR = 70);
CREATE INDEX IF NOT EXISTS variables_variablevalue ON __Variables (VariableValue varchar_pattern_ops) WITH (FILLFACTOR = 70);

SELECT public.ensure_view('RBAC', '__Variables');

-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __Software (
        HostKey                 TEXT                        NOT NULL,
        compo                   TEXT                        NOT NULL,
        SoftwareName            TEXT                        NOT NULL,
        SoftwareVersion         TEXT                        DEFAULT NULL,
        SoftwareArchitecture    TEXT                        NOT NULL,
        ChangeTimeStamp		    timestamp with time zone
    ) WITH (FILLFACTOR = 90);

-- ENT-7419: unknown version should be represented as NULL (but older versions of the schema forbid that)
ALTER TABLE __Software ALTER COLUMN SoftwareVersion DROP NOT NULL;

COMMENT ON TABLE public.__software IS 'Most recent reported state of installed software.';
COMMENT ON COLUMN __software.hostkey IS 'host identifier related to this software inventory. This should have a matching __hosts.hostkey but this is not currently enforced at the database level.';
COMMENT ON COLUMN __software.compo IS 'Combination of software name and software version joined by a dot (.). Used for comparing versions of software and avoiding multiple WHERE clauses.';
COMMENT ON COLUMN __software.softwarename IS 'The name of the software package as understood by the packages promise.';
COMMENT ON COLUMN __software.softwareversion IS 'The version of the software as understood by the packages promise.';
COMMENT ON COLUMN __software.softwarearchitecture IS 'The architecture of the software as understood by the packages promise.';
COMMENT ON COLUMN __software.changetimestamp IS 'Timestamp (GMT) from the perspective of the related host when the most recent change in software state was observed.';

SELECT public.ensure_unique_index('__software_uniq', '__software', 'hostkey, compo', 90);
CREATE INDEX IF NOT EXISTS software_name_version ON __Software (SoftwareName, SoftwareVersion) with (FILLFACTOR = 90);

SELECT public.ensure_view('RBAC', '__Software');

-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __SoftwareUpdates (
        HostKey             TEXT                        NOT NULL,
        compo               TEXT                        NOT NULL,
        PatchName           TEXT                        NOT NULL,
        PatchVersion        TEXT                        NOT NULL,
        PatchArchitecture   TEXT                        NOT NULL,
        PatchReportType     public.PATCH_STATUS                NOT NULL,
        ChangeTimeStamp   	timestamp with time zone
    ) WITH (FILLFACTOR = 90);

SELECT public.ensure_unique_index('__softwareupdates_uniq', '__softwareupdates', 'hostkey, compo', 90);
CREATE INDEX IF NOT EXISTS software_name ON __SoftwareUpdates (PatchName) WITH (FILLFACTOR = 90);

SELECT public.ensure_view('RBAC', '__SoftwareUpdates');

-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __LastSeenHosts (
        HostKey             TEXT                        NOT NULL,
        compo               TEXT                        NOT NULL,
        LastSeenDirection   public.LAST_SEEN_DIRECTION         NOT NULL,
        RemoteHostKey       TEXT                        NOT NULL,
        RemoteHostIP        TEXT,
        LastSeenTimeStamp   timestamp with time zone    NOT NULL,
        LastSeenInterval    real                        NOT NULL
    ) WITH (FILLFACTOR = 70);

SELECT public.ensure_unique_index('__lastseenhosts_uniq', '__lastseenhosts', 'hostkey, compo', 70);
CREATE INDEX IF NOT EXISTS lastseenhosts_remotekey ON __LastSeenHosts (RemoteHostKey) WITH (FILLFACTOR = 70);
SELECT public.ensure_view('RBAC', '__LastSeenHosts');

----------------------Drop __lastseenhostslog table and references-------------
DROP TABLE IF EXISTS  __LastSeenHostsLogs CASCADE;
DROP FUNCTION IF EXISTS  lastseen_hosts_log_function CASCADE;
-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __PromiseExecutions (
        HostKey             TEXT                        NOT NULL,
        PolicyFile          TEXT                        NOT NULL,
        ReleaseId           TEXT                        DEFAULT NULL,
        PromiseHash         TEXT                        NOT NULL,
        NameSpace           TEXT                        NOT NULL,
        BundleName          TEXT                        NOT NULL,
        PromiseType         TEXT                        NOT NULL,
        Promiser            TEXT                        DEFAULT NULL,
        StackPath           TEXT                        NOT NULL,
        PromiseHandle       TEXT                        DEFAULT NULL,
        PromiseOutcome      CHARACTER VARYING(8)        NOT NULL,
        LineNumber          INTEGER                     DEFAULT 0,
        PolicyFileHash      TEXT                        DEFAULT NULL,
        LogMessages         TEXT[]                      DEFAULT NULL,
        Promisees           TEXT[]                      DEFAULT NULL,
        MetaTags            TEXT[]                      DEFAULT NULL,
        ChangeTimeStamp     timestamp with time zone
    ) WITH (FILLFACTOR = 70);

-- change promiseoutcome data type to character varying
DO $$
  BEGIN
    IF NOT EXISTS (
      SELECT 1 from information_schema.columns
      WHERE table_name ILIKE '__promiseexecutions'
        AND table_schema = 'public'
        AND column_name = 'promiseoutcome'
        AND data_type = 'character varying')
      THEN
        DROP VIEW IF EXISTS promiseexecutions;
        ALTER TABLE __PromiseExecutions ALTER COLUMN promiseoutcome type CHARACTER VARYING(8) USING promiseoutcome::text;
    END IF;
END$$;

DO $PUBLIC$
BEGIN
  IF current_schema() = 'public' THEN

-- added later, may need to be added to an existing table on updates
ALTER TABLE __PromiseExecutions ADD COLUMN IF NOT EXISTS LineNumber INTEGER DEFAULT 0;
ALTER TABLE __PromiseExecutions ADD COLUMN IF NOT EXISTS PolicyFileHash TEXT DEFAULT NULL;
ALTER TABLE __PromiseExecutions ADD COLUMN IF NOT EXISTS MetaTags TEXT[] DEFAULT NULL;

COMMENT ON TABLE __promiseexecutions IS 'Promises executed on hosts during their last reported cf-agent run.';
COMMENT ON COLUMN __promiseexecutions.hostkey is 'Unique host identifier. All tables can be joined by HostKey to connect data concerning same hosts.';
COMMENT ON COLUMN __promiseexecutions.policyfile is ' Path to the file where the promise is located.';
COMMENT ON COLUMN __promiseexecutions.releaseid is 'Unique identifier of masterfiles version that is executed on the host.';
COMMENT ON COLUMN __promiseexecutions.promisehash is 'Unique identifier of a promise. It is a hash of all promise attributes and their values.';
COMMENT ON COLUMN __promiseexecutions.namespace is 'Namespace within which the promise is executed. If no namespace is set then it is set as: default.';
COMMENT ON COLUMN __promiseexecutions.bundlename is 'Bundle name where the promise is executed.';
COMMENT ON COLUMN __promiseexecutions.promisetype is 'Type of the promise.';
COMMENT ON COLUMN __promiseexecutions.promiser is 'Object affected by a promise.';
COMMENT ON COLUMN __promiseexecutions.stackpath is 'Call stack of the promise.';
COMMENT ON COLUMN __promiseexecutions.promisehandle is 'A unique id-tag string for referring promise.';
COMMENT ON COLUMN __promiseexecutions.promiseoutcome is ' Promise execution result. (KEPT/NOTKEPT/REPAIRED)';
COMMENT ON COLUMN __promiseexecutions.linenumber is 'The line number in the policy file of the promise.';
COMMENT ON COLUMN __promiseexecutions.policyfilehash is 'Hash of the policy file.';
COMMENT ON COLUMN __promiseexecutions.logmessages is 'List of 5 last messages generated during promise execution. If the promise is KEPT the messages are not reported. Log messages can be used for tracking specific changes made by CFEngine while repairing or failing promise execution.';
COMMENT ON COLUMN __promiseexecutions.promisees is 'List of promisees defined for the promise.';
COMMENT ON COLUMN __promiseexecutions.metatags is 'List of meta tags of the promise.';
COMMENT ON COLUMN __promiseexecutions.changetimestamp is 'The GMT time on the host when cf-agent started for the execution that actuated this promise.';

PERFORM public.ensure_unique_index('__promiseexecutions_uniq', '__promiseexecutions', 'hostkey, promisehash', 70);
CREATE INDEX IF NOT EXISTS promiseexecutions_bundlename ON __PromiseExecutions (BundleName) WITH (FILLFACTOR = 70);
CREATE INDEX IF NOT EXISTS promiseexecutions_promiser ON __PromiseExecutions (Promiser) WITH (FILLFACTOR = 70);
CREATE INDEX IF NOT EXISTS promiseexecutions_promiseoutsome ON __PromiseExecutions (PromiseOutcome) WITH (FILLFACTOR = 70);
CREATE INDEX IF NOT EXISTS promiseexecutions_promisees ON __PromiseExecutions (Promisees) WITH (FILLFACTOR =  70);
CREATE INDEX IF NOT EXISTS promiseexecutions_promisehash ON __PromiseExecutions (PromiseHash) WITH (FILLFACTOR =  70);
PERFORM public.ensure_view('RBAC', '__PromiseExecutions');

END IF;
END $PUBLIC$;

-------------------------------------------------------------------------------
-- Log tables:
-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __ContextsLog (
        HostKey             TEXT                        NOT NULL,
        ChangeTimeStamp     timestamp with time zone    NOT NULL,
        ChangeOperation     public.CHANGE_OPERATION            NOT NULL,
        ContextName         TEXT                        NOT NULL,
        MetaTags            TEXT[]
    );

COMMENT ON TABLE __ContextsLog is 'Classes set on hosts by CFEngine over period of time.';
COMMENT ON COLUMN __ContextsLog.HostKey IS 'Public key digest representing host identity.';
COMMENT ON COLUMN __ContextsLog.ChangeTimeStamp IS 'Timestamp from the perspective of the related host when the class is set in its current form. Note: The statement if true till present time or newer entry claims otherwise.';
COMMENT ON COLUMN __ContextsLog.ChangeOperation IS 'Diff state describing current entry. ADD stands for introducing a new entry which did not exist before. CHANGE indicates a changing value or attribute such as MetaTags. REMOVE stands for a context previously set no longer being set. UNTRACKED indicates information about this context is being filtered and will not report any future information about it.';
COMMENT ON COLUMN __ContextsLog.ContextName IS 'Name of class that was defined.';
COMMENT ON COLUMN __ContextsLog.MetaTags IS 'MetaTags associated with the class.';


SELECT public.ensure_index('contexts_log_hostkey', '__contextslog', 'hostkey', 70, '');

SELECT public.ensure_view('RBAC', '__ContextsLog');

-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __VariablesLog (
        HostKey             TEXT                        NOT NULL,
        ChangeTimeStamp     timestamp with time zone    NOT NULL,
        ChangeOperation     public.CHANGE_OPERATION            NOT NULL,
        NameSpace           TEXT                        DEFAULT NULL,
        Bundle              TEXT                        NOT NULL,
        VariableName        TEXT                        NOT NULL,
        VariableValue       TEXT                        DEFAULT NULL,
        VariableType        TEXT                        NOT NULL,
        MetaTags            TEXT[]
    );

CREATE INDEX IF NOT EXISTS variables_log_hostkey ON __VariablesLog (HostKey) WITH (FILLFACTOR = 70);

SELECT public.ensure_view('RBAC', '__VariablesLog');

-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __SoftwareLog (
        HostKey                 TEXT                        NOT NULL,
        ChangeTimeStamp         timestamp with time zone    NOT NULL,
        ChangeOperation         public.CHANGE_OPERATION            NOT NULL,
        SoftwareName            TEXT                        NOT NULL,
        SoftwareVersion         TEXT                        DEFAULT NULL,
        SoftwareArchitecture    TEXT                        NOT NULL
    );

-- ENT-7419: unknown version should be represented as NULL (but older versions of the schema forbid that)
ALTER TABLE __SoftwareLog ALTER COLUMN SoftwareVersion DROP NOT NULL;

CREATE INDEX IF NOT EXISTS software_log_hostkey ON __SoftwareLog (HostKey) WITH (FILLFACTOR = 70);

SELECT public.ensure_view('RBAC', '__SoftwareLog');

-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __SoftwareUpdatesLog (
        HostKey             TEXT                        NOT NULL,
        ChangeTimeStamp     timestamp with time zone    NOT NULL,
        ChangeOperation     public.CHANGE_OPERATION            NOT NULL,
        PatchName           TEXT                        NOT NULL,
        PatchVersion        TEXT                        NOT NULL,
        PatchArchitecture   TEXT                        NOT NULL,
        PatchReportType     public.PATCH_STATUS                NOT NULL
    );

CREATE INDEX IF NOT EXISTS software_updates_log_hostkey ON __SoftwareUpdatesLog (HostKey) WITH (FILLFACTOR = 70);

SELECT public.ensure_view('RBAC', '__SoftwareUpdatesLog');

-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __FileChangesLog (
        HostKey             TEXT                        NOT NULL,
        PromiseHandle       TEXT                        NOT NULL,
        FileName            TEXT                        NOT NULL,
        ChangeTimeStamp     timestamp with time zone    NOT NULL,
        ChangeType          TEXT                        NOT NULL,
        ChangeDetails       TEXT[]                      DEFAULT NULL
    );

CREATE INDEX IF NOT EXISTS filechanges_hk ON __FileChangesLog (HostKey) WITH (FILLFACTOR = 90);

SELECT public.ensure_view('RBAC', '__FileChangesLog');

-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __BenchmarksLog (
        HostKey             TEXT                        NOT NULL,
        EventName           TEXT                        NOT NULL,
        StandardDeviation   NUMERIC                     NOT NULL,
        AverageValue        NUMERIC                     NOT NULL,
        LastValue           NUMERIC                     NOT NULL,
        CheckTimeStamp      timestamp with time zone    NOT NULL
    );

CREATE INDEX IF NOT EXISTS benchmarkslog_hk ON __BenchmarksLog (HostKey) WITH (FILLFACTOR = 90);

SELECT public.ensure_view('RBAC', '__BenchmarksLog');

-------------------------------------------------------------------------------

-- Drop deprecated __PromiseExecutionsLog table

DROP VIEW IF EXISTS  PromiseExecutionsLog;
DROP TABLE IF EXISTS  __PromiseExecutionsLog;

-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __PromiseLog (
        Id                  bigserial                   primary key,
        HostKey             TEXT                        NOT NULL,
        ChangeTimeStamp     timestamp with time zone    NOT NULL,
        PolicyFile          TEXT                        NOT NULL,
        ReleaseId           TEXT                        DEFAULT NULL,
        PromiseHash         TEXT                        NOT NULL,
        NameSpace           TEXT                        NOT NULL,
        BundleName          TEXT                        NOT NULL,
        PromiseType         TEXT                        NOT NULL,
        Promiser            TEXT                        DEFAULT NULL,
        StackPath           TEXT                        NOT NULL,
        PromiseHandle       TEXT                        DEFAULT NULL,
        PromiseOutcome      public.PROMISE_OUTCOME             NOT NULL,
        LineNumber          INTEGER                     DEFAULT 0,
        PolicyFileHash      TEXT                        DEFAULT NULL,
        LogMessages         TEXT[]                      DEFAULT NULL,
        Promisees           TEXT[]                      DEFAULT NULL,
        MetaTags            TEXT[]                      DEFAULT NULL
    );

-- added later, may need to be added to an existing table on updates
ALTER TABLE __PromiseLog ADD COLUMN IF NOT EXISTS LineNumber INTEGER DEFAULT 0;
ALTER TABLE __PromiseLog ADD COLUMN IF NOT EXISTS PolicyFileHash TEXT DEFAULT NULL;
ALTER TABLE __PromiseLog ADD COLUMN IF NOT EXISTS MetaTags TEXT[] DEFAULT NULL;

COMMENT ON TABLE __PromiseLog IS 'History of NOTKEPT and REPAIRED promises executed on hosts';
COMMENT ON COLUMN __PromiseLog.Id IS 'Automatically incrimented, unique identifier.';
COMMENT ON COLUMN __PromiseLog.HostKey IS 'Unique host identifier. All tables can be joined by HostKey to connect data concerning same hosts.';
COMMENT ON COLUMN __PromiseLog.ChangeTimeStamp IS 'The GMT time on the host when cf-agent started for the execution that actuated this promise.';
COMMENT ON COLUMN __PromiseLog.PolicyFile IS 'Path to the file where the promise is located.';
COMMENT ON COLUMN __PromiseLog.ReleaseId IS 'Unique identifier of masterfiles version that is executed on the host.';
COMMENT ON COLUMN __PromiseLog.PromiseHash IS 'Unique identifier of a promise evaluation.';
COMMENT ON COLUMN __PromiseLog.NameSpace IS 'Namespace within which the promise is executed. If no namespace is set then it is set as: default.';
COMMENT ON COLUMN __PromiseLog.BundleName IS 'Bundle name where the promise is executed.';
COMMENT ON COLUMN __PromiseLog.PromiseType IS 'Type of the promise.';
COMMENT ON COLUMN __PromiseLog.Promiser IS 'Object affected by a promise.';
COMMENT ON COLUMN __PromiseLog.StackPath IS 'Call stack of the promise.';
COMMENT ON COLUMN __PromiseLog.PromiseHandle IS 'A unique id-tag string for referencing the individual promise.';
COMMENT ON COLUMN __PromiseLog.PromiseOutcome IS 'Promise execution result. (KEPT|NOTKEPT|REPAIRED).';
COMMENT ON COLUMN __PromiseLog.LogMessages IS 'List of 5 last messages generated during promise execution. If the promise is KEPT the messages are not reported. Log messages can be used for tracking specific changes made by CFEngine while repairing or failing promise execution.';
COMMENT ON COLUMN __PromiseLog.LineNumber IS 'The line number in the policy file of the promise.';
COMMENT ON COLUMN __PromiseLog.PolicyFileHash IS 'Hash of the policy file.';
COMMENT ON COLUMN __PromiseLog.Promisees IS 'List of promisees defined for the promise.';
COMMENT ON COLUMN __PromiseLog.MetaTags IS 'List of meta tags of the promise.';

CREATE INDEX IF NOT EXISTS promiselog_promiseoutcome ON __PromiseLog (PromiseOutcome);
CREATE INDEX IF NOT EXISTS promiselog_promisehash ON __PromiseLog (PromiseHash);
CREATE INDEX IF NOT EXISTS promiselog_changetimestamp_promiseoutcome ON __promiselog ("changetimestamp", "promiseoutcome");

-- Drop view to change column type to bigint --
DROP VIEW IF EXISTS PromiseLog;
-- Must drop dependent view before altering __PromiseLog
DROP VIEW IF EXISTS not_kept_not_repaired;
ALTER TABLE __PromiseLog ALTER COLUMN id type bigint;

SELECT public.ensure_view('RBAC', '__PromiseLog');

DROP TRIGGER IF EXISTS promise_log_partition_trigger ON __PromiseLog;
CREATE TRIGGER promise_log_partition_trigger
BEFORE INSERT ON __PromiseLog
FOR EACH ROW EXECUTE PROCEDURE public.promise_log_partition_function();

DO $PUBLIC$
BEGIN
  IF current_schema() = 'public' THEN

-- Create empty partitions during installation for 7 days in the past and 3 days up in the future.
PERFORM public.promise_log_partition_create(NOW() - INTERVAL '7 day', 7 + 3, 'REPAIRED');
PERFORM public.promise_log_partition_create(NOW() - INTERVAL '7 day', 7 + 3, 'NOTKEPT');

END IF;
END $PUBLIC$;

-------------------------------------------------------------------------------
-- Monitoring tables:
-------------------------------------------------------------------------------

-- 3.6.2 changes schema for __MonitoringMg, during upgrade old one has to be removed
DO $$
DECLARE
TableExists boolean;
BEGIN
    SELECT INTO TableExists 1 FROM information_schema.tables WHERE table_name = lower('__MonitoringMgMeta');
    IF NOT FOUND THEN
        DROP TABLE IF EXISTS __MonitoringMg;
    END IF;
END
$$ ;


CREATE TABLE IF NOT EXISTS __MonitoringMgMeta (
        id                  bigserial   PRIMARY KEY,
        hostkey             TEXT        NOT NULL,
        observable          TEXT        NOT NULL,
        global              BOOLEAN,
        expected_min        REAL,
        expected_max        REAL,
        unit                TEXT,
        description         TEXT,
        updatedTimeStamp    timestamp with time zone,
        lastUpdatedSample   INTEGER
) WITH (FILLFACTOR = 70);
COMMENT ON TABLE __MonitoringMgMeta IS 'Stores 1 record for ech observable per host.';
COMMENT ON COLUMN __monitoringmgmeta.id IS 'Internal ID for each observable.';
COMMENT ON COLUMN __monitoringmgmeta.hostkey IS 'host identifier related to this observable. This should have a matching __hosts.hostkey but this is not currently enforced at the database level.';
COMMENT ON COLUMN __monitoringmgmeta.observable IS 'The name of the observable. This maps to the promises handle attribute of the measurement promise on the executing client.';
COMMENT ON COLUMN __monitoringmgmeta.global IS 'Waiting on ENT-10394 to be better described.';
COMMENT ON COLUMN __monitoringmgmeta.expected_min IS 'The expected minimum value, but it is unknown whom sets the expectation, cf-monitord?';
COMMENT ON COLUMN __monitoringmgmeta.expected_max IS 'The expected maximum value, but it is unknown whom sets the expectation, cf-monitord?';
COMMENT ON COLUMN __monitoringmgmeta.unit IS 'The unit of the measurement. Used for cosmetic labeling in Mission Portal.';
COMMENT ON COLUMN __monitoringmgmeta.description IS 'A description of the measurement. Used for cosmetic labeling in Mission Portal. Derived from comment as defined by the measurement promise on the executing client.';
COMMENT ON COLUMN __monitoringmgmeta.updatedtimestamp IS 'The time when the values were inserted into the database as observed by cf-hub on the Enterprise Hub. This should match the value of updateedtimestamp where __monitoringmg.sample matches lastupdatedsample but there is no requirement at the database level, ENT-10396.';
COMMENT ON COLUMN __monitoringmgmeta.lastupdatedsample IS 'The ID of the last updated sample. This should have a matching __monitoringmg.sample but this is not required at the database level, ENT-10395.';

-- Drop view to change column type to bigint --
DROP VIEW IF EXISTS MonitoringMgMeta;

DO $PUBLIC$
BEGIN
  IF current_schema() = 'public' THEN

ALTER TABLE __MonitoringMgMeta ALTER COLUMN id type bigint;
PERFORM public.ensure_unique_index('__MonitoringMgMeta_uniq', '__MonitoringMgMeta', 'hostkey, observable', 70);
PERFORM public.ensure_view('RBAC', '__MonitoringMgMeta');
END IF;
END $PUBLIC$;


CREATE TABLE IF NOT EXISTS __MonitoringMg (
        meta_id                bigserial     REFERENCES __MonitoringMgMeta (id) ON DELETE CASCADE,
        sample                 INTEGER,
        value1                 REAL,
        value2                 REAL,
        value3                 REAL,
        value4                 REAL,
        updatedTimeStamp       timestamp with time zone
) WITH (FILLFACTOR = 70);
COMMENT ON TABLE __MonitoringMg IS 'Stores average daily values for each observable over a week. Stores 2016 records per observable per host.';
COMMENT ON COLUMN __monitoringmg.meta_id IS 'Internal ID for each observable. It is required to have a matching __monitoringmgmeta.id.';
COMMENT ON COLUMN __monitoringmg.sample IS 'Internal sample ID. There will be up to 2016 samples per observable.';
COMMENT ON COLUMN __monitoringmg.value1 IS 'The value as measured by cf-monitord on the executing agent.';
COMMENT ON COLUMN __monitoringmg.value2 IS 'The average (aka expected) value as calculated by cf-monitord on the executing agent.';
COMMENT ON COLUMN __monitoringmg.value3 IS 'The standard deviation of the measured value as calculated by cf-monitord on the executing agent.';
COMMENT ON COLUMN __monitoringmg.value4 IS 'The delta (aka difference) between the previously measured value and the currently measured value as calculated by cf-monitord on the executing agent.';

DO $PUBLIC$
BEGIN
  IF current_schema() = 'public' THEN

ALTER TABLE __MonitoringMg ALTER COLUMN meta_id type bigint;
PERFORM public.ensure_unique_index('__MonitoringMg_uniq', '__MonitoringMg', 'meta_id, sample', 70);
CREATE INDEX IF NOT EXISTS monitoring_mg_time ON __MonitoringMg (updatedTimeStamp) WITH (FILLFACTOR = 70);

END IF;
END $PUBLIC$;


-------------------------------------------------------------------------------

-- 3.6.2 changes schema for __MonitoringYr, during upgrade old one has to be removed
DO $$
DECLARE
TableExists boolean;
BEGIN
    SELECT INTO TableExists 1 FROM information_schema.tables WHERE table_name = lower('__MonitoringYrMeta');
    IF NOT FOUND THEN
        DROP TABLE IF EXISTS __MonitoringYr;
    END IF;
END
$$ ;


CREATE TABLE IF NOT EXISTS __MonitoringYrMeta (
        id                  bigserial   PRIMARY KEY,
        hostkey             TEXT        NOT NULL,
        observable          TEXT        NOT NULL,
        global              BOOLEAN,
        expected_min        REAL,
        expected_max        REAL,
        unit                TEXT,
        description         TEXT,
        lastUpdatedSample   INTEGER
) WITH (FILLFACTOR = 70);
COMMENT ON TABLE __MonitoringYrMeta is 'Stores 1 record for each observable per host';

-- Drop view to change column type to bigint --
DROP VIEW IF EXISTS MonitoringYrMeta;

DO $PUBLIC$
BEGIN
  IF current_schema() = 'public' THEN

ALTER TABLE __MonitoringYrMeta ALTER COLUMN id type bigint;

PERFORM public.ensure_unique_index('__MonitoringYrMeta_uniq', '__MonitoringYrMeta', 'hostkey, observable', 70);

PERFORM public.ensure_view('RBAC', '__MonitoringYrMeta');

END IF;
END $PUBLIC$;


CREATE TABLE IF NOT EXISTS __MonitoringYr (
        meta_id                bigserial     REFERENCES __MonitoringYrMeta (id) ON DELETE CASCADE,
        sample                 INTEGER,
        value1                 REAL,
        value2                 REAL,
        value3                 REAL,
        value4                 REAL
) WITH (FILLFACTOR = 70);
COMMENT ON TABLE __MonitoringYr IS 'Stores average weekly values for each observable over 3 years. Stores 156 records per observable per host.';

DO $PUBLIC$
BEGIN
  IF current_schema() = 'public' THEN

ALTER TABLE __MonitoringYr ALTER COLUMN meta_id type bigint;

PERFORM public.ensure_unique_index('__MonitoringYr_uniq', '__MonitoringYr', 'meta_id, sample', 70);


END IF;
END $PUBLIC$;

-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __MonitoringHg (
        host    TEXT        NOT NULL,
        id      TEXT        NOT NULL,
        ar1     REAL[]
    ) WITH (FILLFACTOR = 70);
COMMENT ON TABLE __MonitoringHg IS 'Stores 1 record for each observable per host.';

SELECT public.ensure_unique_index('__MonitoringHg_uniq', '__MonitoringHg', 'host, id', 70);

SELECT public.ensure_view('RBAC', '__MonitoringHg', 'MonitoringHg', 'host');

-------------------------------------------------------------------------------
-- Diagnostics tables:
-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS __HubConnectionErrors (
        CheckTimeStamp      timestamp with time zone    DEFAULT CURRENT_TIMESTAMP,
        HostKey             TEXT                        NOT NULL,
        Message             TEXT                        NOT NULL,
        QueryType           TEXT                        NOT NULL
    );

SELECT public.ensure_view('RBAC', '__HubConnectionErrors');

-------------------------------------------------------------------------------

-- 3.6.2 Renames '__Diagnostics' to 'Diagnostics' table
ALTER TABLE IF EXISTS __Diagnostics RENAME TO Diagnostics;


CREATE TABLE IF NOT EXISTS Diagnostics (
        Name           TEXT                         NOT NULL,
        Details        TEXT,
        TimeStamp      timestamp with time zone     DEFAULT CURRENT_TIMESTAMP,
        Value          NUMERIC                      NOT NULL,
        Units          TEXT                         NOT NULL
    );

-- ENT-4331: 3.14.0/3.12.2 renames 'Status' to '__Status' and creates a RBAC-checked view
-- ENT-5396: Upgrades from 3.12.1 to 3.12.2 and 3.12.3 leave us with both __status and status tables.
-- we must ensure things are correct and merge data if need be.
DO $$
BEGIN
  IF current_schema() = 'public' THEN
    IF verify_table('__status') AND verify_table('status') THEN
      RAISE NOTICE 'Found both status and __status tables. Migrating all data to __status to make proper.';
      CREATE TABLE tmp_status (LIKE __status);
      INSERT INTO tmp_status SELECT * FROM __status;
      INSERT INTO tmp_status SELECT * FROM status;
      DROP TABLE __status;
      DROP TABLE status;
      ALTER TABLE tmp_status RENAME TO __status;
    ELSIF public.verify_table('status') THEN
      ALTER TABLE status RENAME to __status;
    END IF;
  END IF;
END $$;
    
CREATE TABLE IF NOT EXISTS __STATUS (
        host           CHARACTER VARYING(256)   NOT NULL,
        ts             CHARACTER VARYING(256)   NOT NULL,
        hubts          timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP,
        status         CHARACTER VARYING(256),
        lstatus        CHARACTER VARYING(256),
        type           CHARACTER,
        who            INTEGER,
        whr            INTEGER,
        PRIMARY        KEY(host,ts)
    ) WITH (FILLFACTOR = 90);

DO $PUBLIC$
BEGIN
  IF current_schema() = 'public' THEN

ALTER TABLE __STATUS
ADD COLUMN IF NOT EXISTS hubts timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP;

COMMENT ON TABLE __STATUS IS 'Report collection status';
COMMENT ON COLUMN __STATUS.ts IS 'Timestamp of last data provided by client during report collection. This is used by delta queries to request a start time.';

PERFORM public.ensure_view('RBAC', '__Status', 'Status', 'host');

END IF;
END $PUBLIC$;


-------------------------------------------------------------------------------

DO $SPECIAL$
BEGIN
IF (public.is_superhub() AND current_schema() != 'public') OR NOT public.is_superhub() THEN

-- Update ContextCache after all tables are created (Useful during hub upgrade)
-- Warning: order is important.

TRUNCATE TABLE ContextCache;

INSERT INTO ContextCache (hostkey, contextvector)
    SELECT hostkey, to_tsvector('simple', translate(x::text,'_,:','.,.'))
    FROM (
        SELECT hostkey, array_agg(contextname) as x
        FROM __Contexts
        GROUP BY hostkey
        ) as sub;

INSERT INTO ContextCache(hostkey, contextvector)
    SELECT hostkey, to_tsvector('simple',replace('host_not_reported'::text,'_','.'))
    FROM __hosts
    WHERE hostkey NOT IN (SELECT hostkey FROM ContextCache);

END IF;
END $SPECIAL$;

DO $PUBLIC$
BEGIN
IF current_schema() = 'public' THEN


----------------------------------------------------
--- ENT-4554 Change m_inventory into a regular table
----------------------------------------------------

-- Drop old inventory views and function --
DO $$
DECLARE
  record_count integer;
BEGIN

  SELECT count(*)
  INTO record_count
  FROM pg_matviews
  WHERE matviewname = 'inventory_new';

  IF record_count = 1 then
    EXECUTE 'DROP MATERIALIZED VIEW public.inventory_new';
  ELSE
    EXECUTE 'DROP VIEW IF EXISTS public.inventory_new';
  END IF;

END $$;

DROP MATERIALIZED VIEW IF EXISTS public.m_inventory;
DROP FUNCTION IF EXISTS update_materialized_inventory_view;

CREATE TABLE IF NOT EXISTS  __inventory (
    hostkey text PRIMARY KEY,
    values jsonb NOT NULL
) WITH (FILLFACTOR = 90);

COMMENT ON TABLE "__inventory" IS 'Stores normalized inventory data';

PERFORM public.ensure_view('RBAC', '__inventory', 'inventory_new');


COMMENT ON VIEW "inventory_new" IS 'Regular view based on __inventory table with applied RBAC filter';

-- In postgresql 12.1 `DROP FUNCTION IF EXISTS` was changed to give
-- an ERROR if there is a matching procedure of the same name. Check and drop manually.
DO $$
BEGIN
  IF verify_function('update_inventory') THEN
    DROP FUNCTION IF EXISTS update_inventory();
  END IF;
END$$;

DROP PROCEDURE IF EXISTS update_inventory();
CREATE PROCEDURE update_inventory()
  LANGUAGE plpgsql
  AS $$
BEGIN
  IF pg_is_in_recovery() THEN
    RAISE LOG 'PostgreSQL is in recovery, skipping update of __inventory table';
  ELSIF public.is_superhub() THEN
    RAISE DEBUG 'superhub is enabled, __inventory will not be updated';
  ELSE
    RAISE LOG 'Updating __inventory table';
    DROP TABLE IF EXISTS __tmp_inventory;
    CREATE TABLE __tmp_inventory (LIKE __inventory INCLUDING ALL);
    INSERT INTO __tmp_inventory (SELECT * FROM v_inventory WHERE values IS NOT NULL);
    TRUNCATE __inventory;
    INSERT INTO __inventory SELECT * FROM __tmp_inventory;
    DROP TABLE __tmp_inventory;
  END IF;
END
$$;
COMMENT ON PROCEDURE update_inventory IS 'Update __inventory table';

CREATE OR REPLACE FUNCTION ensure_v_inventory_view() RETURNS VOID AS $body$
DECLARE hosts_hub_id_column TEXT;
BEGIN
  IF public.is_superhub() THEN
    hosts_hub_id_column = ', __hosts.hub_id ';
  END IF;

  EXECUTE format('
CREATE OR REPLACE VIEW public.v_inventory AS
SELECT __hosts.hostkey,
( SELECT json_object(array_agg(d.keyname), array_agg(d.value)) AS array_to_json
  FROM (
    SELECT  keyname,
    string_agg(cf_clearslist(value),'', '') AS value
    FROM( SELECT substring(__variables.metatags::TEXT FROM ''attribute_name=["]?([^"},]+)'') AS keyname,
      string_agg(cf_clearslist(__variables.variablevalue),'', '')   AS value
      FROM __variables
      WHERE __variables.hostkey = __hosts.hostkey
      AND (''inventory''::text = ANY (__variables.metatags)) AND (__variables.metatags::text LIKE ''%%attribute_name%%'') AND NOT (''attribute_name=none''::text = ANY (__variables.metatags))
      GROUP BY __variables.metatags

      UNION ALL

      SELECT substring(__contexts.metatags::TEXT FROM ''attribute_name=["]?([^"},]+)'') AS keyname,
      string_agg(cf_clearslist(__contexts.contextname),'', '') AS value
      FROM __contexts
      WHERE __contexts.hostkey = __hosts.hostkey
      AND (''inventory''::text = ANY (__contexts.metatags)) AND (__contexts.metatags::text LIKE ''%%attribute_name%%'') AND NOT (''attribute_name=none''::text = ANY (__contexts.metatags))
      GROUP BY __contexts.metatags)f  GROUP BY keyname
    )d
  ) AS "values"
%s -- __hosts.hub_id
FROM __hosts;', hosts_hub_id_column);

EXECUTE 'CALL update_inventory()'; -- because we may have just added hub_id column

COMMENT ON VIEW "v_inventory" IS 'Regular view to get normalized inventory data';
END;
$body$ LANGUAGE plpgsql;

PERFORM public.ensure_view('FUNCTION', 'ensure_v_inventory_view');

-- postgresql 12.1 workaround for DROP FUNCTION when a procedure of same
-- signature exists.
DO $$
BEGIN
  IF verify_function('update_inventory_by_hostkey',1) THEN
    DROP FUNCTION IF EXISTS update_inventory_by_hostkey(hostkey TEXT);
  END IF;
END$$;

DROP PROCEDURE IF EXISTS update_inventory_by_hostkey(hostkey TEXT);
CREATE PROCEDURE update_inventory_by_hostkey(hostkey TEXT)
  LANGUAGE plpgsql
  AS $$
BEGIN
  IF pg_is_in_recovery() THEN
    RAISE LOG 'PostgreSQL is in recovery, skipping update of __inventory table';
  ELSE
    RAISE DEBUG 'Begin updating __inventory for hostkey %', hostkey;
    -- only superhub should be reporting to the superhub itself
    -- switch to superhub-specific schema to avoid deadlocks with import process
    EXECUTE 'SELECT use_default_hub_schema()';
    EXECUTE 'DELETE FROM __inventory WHERE hostkey = ''' || hostkey || '''';
    EXECUTE 'INSERT INTO __inventory (SELECT * FROM public.parse_inventory_by_hostkey('''|| hostkey ||''') WHERE hostkey IS NOT NULL)';
    RAISE DEBUG 'Done updating __inventory for hostkey %', hostkey;
  END IF;
END
$$;

COMMENT ON PROCEDURE update_inventory_by_hostkey IS 'Update specific host inventory data';

DROP FUNCTION IF EXISTS parse_inventory_by_hostkey(TEXT);
CREATE OR REPLACE FUNCTION parse_inventory_by_hostkey(hostkey TEXT)
  RETURNS __inventory AS $$
DECLARE
  row __inventory%rowtype;
  inventoryViewDefinition TEXT;
  hostExists int;
BEGIN
EXECUTE 'SELECT 1 FROM __hosts WHERE hostkey = '''|| hostkey ||'''' INTO hostExists;
  IF hostExists = 1 THEN
      EXECUTE 'SELECT pg_get_viewdef(''public.v_inventory'', true)' INTO inventoryViewDefinition;
      inventoryViewDefinition = RTRIM(inventoryViewDefinition, ';');
      EXECUTE inventoryViewDefinition || ' WHERE __hosts.hostkey = '''|| hostkey ||'''' INTO row;
      IF row.values IS NULL THEN
          -- if there is no inventory json values then set empty object --
         row.values = '{}';
      END IF;
      RETURN row;
  ELSE
      RAISE NOTICE 'Host % not found. Notice raised while parse_inventory_by_hostkey function performing', hostkey;
      RETURN NULL;
  END IF;
END;
$$ LANGUAGE plpgsql;

COMMENT ON FUNCTION parse_inventory_by_hostkey IS 'Parse specific host inventory data from __variables and __contexts';

-------------------------------------------------------------------------------
-- Hosts view
-------------------------------------------------------------------------------

-- Create Hosts view
CREATE OR REPLACE FUNCTION ensure_hosts_views() RETURNS VOID AS $body$
DECLARE hosts_hub_id_text TEXT;
        and_inventory_hub_id_text TEXT;
BEGIN
  IF public.is_superhub() THEN
    hosts_hub_id_text = ', __hosts.hub_id';
    and_inventory_hub_id_text = ' AND inv.hub_id=__hosts.hub_id';
  END IF;
  EXECUTE 'DROP VIEW IF EXISTS hosts CASCADE';
  EXECUTE format('
CREATE VIEW hosts AS
(SELECT
        __hosts.hostkey,
        inv.values->>''Host name'' as hostname,
        __hosts.ipaddress,
        __hosts.lastreporttimestamp,
        __hosts.firstreporttimestamp,
        __hosts.hostkeycollisions
        %s
FROM __hosts
LEFT JOIN __inventory AS inv ON inv.hostkey = __hosts.hostkey %s
WHERE  (
    CASE WHEN
      coalesce(current_setting(''rbac.filter'', true), '''') = ''''
    THEN
      __hosts.hostkey IS NOT NULL
    ELSE
      __hosts.hostkey IN (SELECT * FROM get_rbac_hostkeys())
    END ) AND __hosts.deleted IS NULL
);', hosts_hub_id_text, and_inventory_hub_id_text);
  ALTER TABLE public.hosts OWNER TO cfapache;
END;
$body$ LANGUAGE plpgsql;

PERFORM public.ensure_view('FUNCTION', 'ensure_hosts_views');

-- Manage hosts_insert_trigger and hosts_delete_trigger
DO $$
DECLARE
  target_schema TEXT default 'public';
BEGIN
  IF public.is_superhub() THEN
    target_schema := 'hub_0';
  END IF;

  EXECUTE format('
DROP TRIGGER IF EXISTS hosts_insert_trigger ON %s.__hosts;
DROP TRIGGER IF EXISTS hosts_delete_trigger ON %s.__hosts;
', target_schema, target_schema, target_schema);
END$$;

-- DROP to allow for new function signature:
DROP FUNCTION IF EXISTS clear_hosts_references() CASCADE;
DROP FUNCTION IF EXISTS clear_hosts_references(TEXT) CASCADE;
-- Clear hosts references in case of a host was deleted
CREATE FUNCTION clear_hosts_references(target_hostkey TEXT)
RETURNS VOID AS
$BODY$
BEGIN
    DELETE FROM contextcache WHERE hostkey = target_hostkey;
    DELETE FROM __agentstatus WHERE hostkey = target_hostkey;
    DELETE FROM __benchmarkslog WHERE hostkey = target_hostkey;
    DELETE FROM __contexts WHERE hostkey = target_hostkey;
    DELETE FROM __contextslog WHERE hostkey = target_hostkey;
    DELETE FROM __cmdb WHERE hostkey = target_hostkey;
    DELETE FROM __filechangeslog WHERE hostkey = target_hostkey;
    DELETE FROM __health_diagnostics_dismissed WHERE hostkey = target_hostkey;
    DELETE FROM __hubconnectionerrors WHERE hostkey = target_hostkey;
    DELETE FROM __inventory WHERE hostkey = target_hostkey;
    DELETE FROM __lastseenhosts WHERE hostkey = target_hostkey;
    DELETE FROM __monitoringyrmeta WHERE hostkey = target_hostkey;
    DELETE FROM __monitoringmgmeta WHERE hostkey = target_hostkey;
    DELETE FROM __monitoringhg WHERE host = target_hostkey;
    DELETE FROM __promiseexecutions WHERE hostkey = target_hostkey;
    DELETE FROM __promiselog WHERE hostkey = target_hostkey;
    DELETE FROM __software WHERE hostkey = target_hostkey;
    DELETE FROM __softwarelog WHERE hostkey = target_hostkey;
    DELETE FROM __softwareupdates WHERE hostkey = target_hostkey;
    DELETE FROM __softwareupdateslog WHERE hostkey = target_hostkey;
    DELETE FROM __status WHERE host = target_hostkey;
    DELETE FROM __variables WHERE hostkey = target_hostkey;
    DELETE FROM __variableslog WHERE hostkey = target_hostkey;
END;
$BODY$
LANGUAGE plpgsql;

DROP FUNCTION IF EXISTS remove_feeder_data(integer);
CREATE FUNCTION remove_feeder_data(target_hub_id integer)
    RETURNS VOID AS
$BODY$
BEGIN
    EXECUTE format('DELETE FROM public.__hubs WHERE hub_id = %s;', target_hub_id);
    EXECUTE format('DROP SCHEMA IF EXISTS hub_%s CASCADE', target_hub_id);
    EXECUTE format('DROP SCHEMA IF EXISTS hub_%s_importing CASCADE', target_hub_id);
END;
$BODY$
LANGUAGE plpgsql;

DO $$
DECLARE
  target_schema TEXT default 'public';
BEGIN
  IF public.is_superhub() THEN
    target_schema := 'hub_0';
  END IF;

  EXECUTE format('
DROP TRIGGER IF EXISTS hosts_clear_refrences_trigger ON %s.__hosts;
', target_schema);
END$$;

-- DROP VIEW public.v_contextcache
DROP VIEW IF EXISTS public.v_contextcache;

-- Create TABLE public.variables_dictionary
CREATE TABLE IF NOT EXISTS public.variables_dictionary
(
  id serial   PRIMARY KEY,
  attribute_name character varying(200),
  category character varying(200),
  readonly integer DEFAULT 0,
  type character varying(200),
  convert_function character varying(200),
  keyname character varying(200),
  CONSTRAINT variables_dictionary_attribute_name UNIQUE (attribute_name)
)
WITH (
  OIDS=FALSE
);

ALTER TABLE variables_dictionary
ADD COLUMN IF NOT EXISTS enabled integer DEFAULT 0;

ALTER TABLE public.variables_dictionary OWNER TO cfpostgres; -- squashed to make unit testing easier
GRANT SELECT ON TABLE public.variables_dictionary TO public;
GRANT ALL ON TABLE public.variables_dictionary TO cfpostgres;


-------------------------------------------------------------------------------
-- Events table :
-------------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS  events (
    id        bigserial   PRIMARY KEY,
    event character varying(30) NOT NULL,
    payload json DEFAULT '{}' NOT NULL,
    date timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL
) WITH (FILLFACTOR = 90);

-------------------------------------------------------------------------------
-- Function and trigger to handle decommissioned and bootstrapped hosts events :
-------------------------------------------------------------------------------

CREATE OR REPLACE FUNCTION process_host_events() RETURNS TRIGGER AS $host_log_event_trigger$
BEGIN
  IF pg_is_in_recovery() THEN
    RAISE LOG 'PostgreSQL is in recovery, skipping processing host events';
  ELSE
    RAISE NOTICE 'Processing host events';
    IF (TG_OP = 'UPDATE' AND NEW.deleted IS NOT NULL AND OLD.deleted IS NULL) THEN
      INSERT INTO events  (event, payload) VALUES ('host_decommissioned', json_build_object('hostkey', OLD.hostkey));
      RETURN OLD;
    ELSIF (TG_OP = 'INSERT') THEN
      INSERT INTO events  (event, payload) VALUES ('host_bootstrapped', json_build_object('hostkey', NEW.hostkey));
      RETURN NEW;
    END IF;
  END IF;
  RETURN NULL;
END;
$host_log_event_trigger$ LANGUAGE plpgsql;

-- Manage host_log_event_trigger
DO $$
DECLARE
  target_schema TEXT default 'public';
BEGIN
  IF public.is_superhub() THEN
    target_schema := 'hub_0';
  END IF;

  EXECUTE format('
DROP TRIGGER IF EXISTS host_log_event_trigger ON %s.__hosts;
CREATE TRIGGER host_log_event_trigger
AFTER INSERT OR UPDATE ON %s.__hosts
    FOR EACH ROW EXECUTE PROCEDURE process_host_events();
',
  target_schema, target_schema);
END$$;

---- ENT-3950 ----

CREATE TABLE IF NOT EXISTS "__health_diagnostics_dismissed" (
    hostkey text,
    report_type character varying(30),
    username character varying(20),
    CONSTRAINT "health_diagnostics_dismissed_hostkey_report_type_username" PRIMARY KEY ("hostkey", "report_type", "username")
);

COMMENT ON TABLE "__health_diagnostics_dismissed" IS 'Table that contains specific dismissed hosts from Health diagnostic reports';

PERFORM public.ensure_view('RBAC', '__health_diagnostics_dismissed', 'health_diagnostics_dismissed', 'hostkey');

---- ENT-4567 Add __hub table to store hub's information ----
CREATE TABLE IF NOT EXISTS __hub (
    "hostkey" text PRIMARY KEY,
    "hostname" text,
    "ip" text,
    "update_ts" timestamptz
);

COMMENT ON TABLE "__hub" IS 'Table stores hub''s information';

-- Manage privileges on events_id_seq and events table
DO $$
DECLARE
  target_schema TEXT default 'public';
BEGIN
  IF public.is_superhub() THEN
    target_schema := 'hub_0';
  END IF;

  EXECUTE format('
GRANT ALL PRIVILEGES ON SEQUENCE %s.events_id_seq TO public;
GRANT ALL PRIVILEGES ON TABLE %s.events TO public;
',
  target_schema, target_schema);
END$$;

---- ENT-4611 schema changes on enabling superhub ----

CREATE OR REPLACE FUNCTION ensure_views() RETURNS VOID AS $body$
DECLARE r record;
BEGIN
  FOR r in SELECT * FROM public.__Views ORDER BY Id
  LOOP
    EXECUTE 'SELECT ensure_view($1,$2,$3,$4)' USING r.viewtype, r.sourcename, r.viewname, r.hostkeycolumn;
  END LOOP;
END;
$body$ LANGUAGE plpgsql;

CREATE OR REPLACE function switch_to_feeder_schema(hostkey_search TEXT) RETURNS VOID AS $$
DECLARE id NUMERIC;
        feeder_schema TEXT;
BEGIN
  id := get_hub_id(hostkey_search);
  feeder_schema = 'hub_' || id || '_importing';

  RAISE DEBUG 'set schema %', feeder_schema;
  EXECUTE 'SET SCHEMA ' || quote_literal(feeder_schema);
END;
$$ LANGUAGE plpgsql;

CREATE OR REPLACE function obtain_shared_schema_lock() RETURNS VOID AS $$
BEGIN
  PERFORM public.obtain_schema_lock(true);
END;
$$ LANGUAGE plpgsql;

CREATE OR REPLACE function obtain_exclusive_schema_lock() RETURNS VOID AS $$
BEGIN
  PERFORM public.obtain_schema_lock(false);
END;
$$ LANGUAGE plpgsql;

CREATE OR REPLACE function obtain_schema_lock(shared BOOLEAN = false) RETURNS VOID AS $$
DECLARE lock_timestamp TEXT;
        lock_type TEXT;
BEGIN
  IF shared THEN
    lock_type := 'shared';
  ELSE
    lock_type := 'exclusive';
  END IF;

  RAISE DEBUG 'wait for % schema lock...', lock_type;

  IF shared THEN
    PERFORM pg_advisory_lock_shared(18002);
  ELSE
    PERFORM pg_advisory_lock(18002);
  END IF;

  PERFORM set_config('global.schema_lock_timestamp', EXTRACT(EPOCH FROM clock_timestamp())::TEXT, false);
  SELECT current_setting('global.schema_lock_timestamp', true) INTO lock_timestamp;
  RAISE DEBUG '% schema lock was obtained with global.schema_lock_timestamp value %', lock_type, lock_timestamp;
END;
$$ LANGUAGE plpgsql;

DROP FUNCTION IF EXISTS release_shared_schema_lock();
CREATE OR REPLACE function release_shared_schema_lock(purpose TEXT default 'unknown') RETURNS VOID AS $$
BEGIN
  PERFORM public.release_schema_lock(true, purpose);
END;
$$ LANGUAGE plpgsql;

DROP FUNCTION IF EXISTS release_exclusive_schema_lock();
CREATE OR REPLACE function release_exclusive_schema_lock(purpose TEXT default 'unknown') RETURNS VOID AS $$
BEGIN
  PERFORM public.release_schema_lock(false, purpose);
END;
$$ LANGUAGE plpgsql;

DROP FUNCTION IF EXISTS release_schema_lock(BOOLEAN);
CREATE OR REPLACE function release_schema_lock(shared BOOLEAN, purpose TEXT) RETURNS VOID AS $$
DECLARE lock_type TEXT;
        lock_timestamp TEXT;
        lock_duration NUMERIC;
BEGIN
  IF shared THEN
    lock_type := 'shared';
  ELSE
    lock_type := 'exclusive';
  END IF;

  -- second param is "missing ok" and will return NULL if not there, aka the lock wasn't obtained in this session yet
  SELECT current_setting('global.schema_lock_timestamp', true) INTO lock_timestamp;
  IF lock_timestamp = NULL or lock_timestamp = '' THEN
    RAISE DEBUG '% schema lock was not obtained yet or cleared during an exception for this session', lock_type;
    RETURN;
  END IF;
  IF lock_timestamp = 'RELEASED' THEN
    RAISE DEBUG '% schema lock already released in this session', lock_type;
    RETURN;
  END IF;

  IF shared THEN
    PERFORM pg_advisory_unlock_shared(18002);
  ELSE
    PERFORM pg_advisory_unlock(18002);
  END IF;

  PERFORM set_config('global.schema_lock_timestamp', 'RELEASED', false);
  lock_duration := EXTRACT(EPOCH FROM clock_timestamp()) - EXTRACT(EPOCH FROM to_timestamp(lock_timestamp::NUMERIC));
  RAISE DEBUG 'released % schema lock 18002 held for % seconds for purpose %', lock_type, lock_duration, purpose;
END;

$$ LANGUAGE plpgsql;


CREATE OR REPLACE function drop_feeder_schema(hostkey_search TEXT) RETURNS VOID AS $$
DECLARE id NUMERIC;
        feeder_schema TEXT;
BEGIN
  PERFORM public.obtain_exclusive_schema_lock();

  id := get_hub_id(hostkey_search);
  feeder_schema = 'hub_' || id || '_importing';

  RAISE DEBUG 'dropping schema %', feeder_schema;
  EXECUTE 'DROP SCHEMA ' || feeder_schema || ' CASCADE';

  PERFORM public.release_exclusive_schema_lock('drop_feeder_schema ' || feeder_schema);
EXCEPTION WHEN others THEN
  PERFORM public.release_exclusive_schema_lock('exception in drop_feeder_schema ' || feeder_schema);
  RAISE NOTICE 'exception in drop_feeder_schema % - %', sqlstate, sqlerrm;
END;
$$ LANGUAGE plpgsql;

CREATE OR REPLACE function ensure_feeder_schema(hostkey_search TEXT, table_whitelist TEXT[]) RETURNS VOID AS $$
DECLARE id NUMERIC;
        _tablename TEXT;
        feeder_table TEXT;
        new_schema TEXT;
BEGIN
  PERFORM public.obtain_exclusive_schema_lock();

  id := get_hub_id(hostkey_search);
  new_schema = 'hub_' || id || '_importing';

  -- renaming the schema will not affect use of it, everything should still work as before
  IF verify_schema(new_schema) THEN
    EXECUTE 'DROP SCHEMA ' || new_schema || ' CASCADE'; -- get rid of old import schema, shouldn't happen
  END IF;

  -- create the new schema in which to import feeder data
  EXECUTE 'CREATE SCHEMA ' || new_schema;
  EXECUTE 'SET SCHEMA ' || quote_literal(new_schema);

  FOREACH _tablename IN ARRAY table_whitelist
  LOOP
    RAISE DEBUG 'handling whitelist entry %', _tablename;
    -- new_schema __promiselog is a temporary table to handle import
    -- will be removed and have child tables re-attached to public.__promiselog
    -- in attach_feeder_schema().
    IF _tablename = '__promiselog_*' THEN
      _tablename := '__promiselog';
    END IF;

    feeder_table := new_schema || '.' || _tablename;

    RAISE DEBUG 'create table % like public.%', feeder_table, _tablename;
    EXECUTE 'CREATE TABLE ' || feeder_table || ' (LIKE public.' || _tablename || ' INCLUDING DEFAULTS INCLUDING CONSTRAINTS)';

    RAISE DEBUG 'setting default hub_id of table % to %', feeder_table, id;
    EXECUTE 'ALTER TABLE ' || feeder_table || ' ALTER COLUMN hub_id SET DEFAULT ' || id;

    IF _tablename = '__promiselog' THEN
      RAISE DEBUG 'adding feeder schema trigger for __promiselog';
      EXECUTE 'CREATE TRIGGER feeder_promise_log_partition_trigger BEFORE INSERT ON ' || feeder_table || ' FOR EACH ROW EXECUTE PROCEDURE public.promise_log_partition_function()';
    END IF;

  END LOOP;

  RAISE DEBUG 'set schema %', new_schema;
  EXECUTE 'SET SCHEMA ''' || new_schema || '''';

  PERFORM public.release_exclusive_schema_lock('ensure_feeder_schema ' || hostkey_search);
  RAISE DEBUG 'done with ensure_feeder_schema, no exceptions';
EXCEPTION WHEN others THEN
  PERFORM public.release_exclusive_schema_lock('exception in ensure_feeder_schema ' || hostkey_search);
  RAISE NOTICE 'ensure_feeder_schema had an exception % - %', sqlstate, sqlerrm;
END;
$$ LANGUAGE plpgsql;



-------------------------------------------------------------------------------
-- handle_duplicate_hostkeys_in_import()
-------------------------------------------------------------------------------
-- Provided an array of hostkeys for the current set of importing feeders
-- find duplicate hostkeys in hub_<n> and hub_<n>_importing schemas.
-- Choose the entry with the newest __hosts.lastreporttimestamp and use
-- that set of data (either public or one of the feeders schemas).
-- Move all other non-logging table data to a dup schema for later analysis.
-- Ensure that any records in dup are unique and updated each run.
CREATE OR REPLACE FUNCTION handle_duplicate_hostkeys_in_import(importing_feeder_hostkeys TEXT[]) RETURNS VOID AS $$
DECLARE tmp_schema TEXT;
        tmp_table TEXT;
        dup_schema TEXT;
        dup_table TEXT;
        source_schema TEXT; -- for moving duplicates to either importing or normal feeder schemas
        tables TEXT[] = ARRAY[
            'contextcache',
            '__agentstatus',
            '__contexts',
            '__hosts',
            '__inventory',
            '__lastseenhosts',
            '__promiseexecutions',
            '__software',
            '__softwareupdates',
            '__variables'
        ];
        schemas_union_sql TEXT = '';
        _table TEXT;
        rec RECORD;
        dup_count int := 0;
        feeder_hub_id BIGINT;
        importing_hub_ids BIGINT[];
        not_importing_hub_ids BIGINT[];
        error_code TEXT;
        error_message TEXT;
        error_stack TEXT;
BEGIN
  PERFORM public.obtain_exclusive_schema_lock();

  RAISE DEBUG '% Begin handling duplicate hostkeys in import', clock_timestamp();
  dup_schema = 'dup';
  tmp_schema = 'tmp';

  RAISE DEBUG 'Ensuring temporary and duplicate schemas exist';
  IF verify_schema(tmp_schema) THEN
    EXECUTE 'DROP SCHEMA ' || tmp_schema || ' CASCADE';
  END IF;
  EXECUTE 'CREATE SCHEMA ' || tmp_schema;
  IF verify_schema(dup_schema) THEN
    EXECUTE 'DROP SCHEMA ' || dup_schema || ' CASCADE';
  END IF;
  EXECUTE 'CREATE SCHEMA ' || dup_schema;

  RAISE DEBUG 'Determining if feeders are importing or not';
  FOR rec in SELECT hostkey, hub_id from __hubs WHERE hub_id != 0
  LOOP
    IF importing_feeder_hostkeys @> ARRAY[rec.hostkey] THEN
      RAISE DEBUG 'Adding feeder %, hub_id % to importing_hub_ids', rec.hostkey, rec.hub_id;
      importing_hub_ids := array_append(importing_hub_ids, rec.hub_id);
    ELSE
      IF verify_schema('hub_' || rec.hub_id) THEN
        schemas_union_sql := schemas_union_sql || ' UNION ALL SELECT * FROM hub_' || rec.hub_id || '.__hosts ';
      END IF;
      RAISE DEBUG 'Adding feeder %, hub_id % to NOT importing hub ids', rec.hostkey, rec.hub_id;
      not_importing_hub_ids := array_append(not_importing_hub_ids, rec.hub_id);
    END IF;
  END LOOP;

  RAISE DEBUG 'Create tables in temporary and duplicate schemas';
  FOREACH _table in ARRAY tables
  LOOP
    tmp_table := tmp_schema || '.' || _table;
    dup_table := dup_schema || '.' || _table;
    EXECUTE 'CREATE TABLE ' || tmp_table || ' (LIKE public.' || _table || ' INCLUDING DEFAULTS) PARTITION BY LIST (hub_id)';
    EXECUTE 'CREATE TABLE ' || dup_table || ' (LIKE public.' || _table || ' INCLUDING DEFAULTS)';

    RAISE DEBUG 'Attaching importing tables to % schema', tmp_schema;
    FOREACH feeder_hub_id in ARRAY importing_hub_ids
    LOOP
      EXECUTE 'ALTER TABLE ' || tmp_table || ' ATTACH PARTITION hub_' || feeder_hub_id || '_importing.' || _table || ' FOR VALUES IN (' || feeder_hub_id || ')';
    END LOOP;
  END LOOP;

  RAISE DEBUG '% Find duplicates and move to appropriate schema', clock_timestamp();
  tmp_table := tmp_schema || '.__hosts';
  EXECUTE '
    CREATE TEMP TABLE duplicate_hostkeys AS
    SELECT
      hostkey,
      hub_id
    FROM (
      SELECT *,row_number()
        OVER (
          PARTITION BY hostkey
          ORDER BY lastreporttimestamp DESC
        ) as rnum
      FROM (
        SELECT * FROM ' || tmp_table || schemas_union_sql || '
      ) cte
    ) cte2
  WHERE cte2.rnum > 1';

  SELECT count(*) FROM duplicate_hostkeys INTO dup_count;
  RAISE INFO 'Found % duplicates. All duplicate data has been moved to schema "%" for analysis.', dup_count, dup_schema;

  FOR rec IN SELECT DISTINCT(hub_id) as hub_id FROM duplicate_hostkeys
  LOOP
    RAISE DEBUG '% Processing duplicates for hub_id %', clock_timestamp(), rec.hub_id;
    EXECUTE 'CREATE TEMP TABLE duplicates_' || rec.hub_id || ' AS SELECT hostkey FROM duplicate_hostkeys WHERE hub_id = ' || rec.hub_id;
    EXECUTE 'CREATE TEMP TABLE deletes_' || rec.hub_id || ' AS SELECT d1.hostkey FROM dup.__hosts as d1, duplicates_' || rec.hub_id || ' AS d2 WHERE d1.hostkey = d2.hostkey';

    IF ARRAY[rec.hub_id] <@ importing_hub_ids THEN
      source_schema := 'hub_' || rec.hub_id || '_importing';
    ELSE
      source_schema := 'hub_' || rec.hub_id;
    END IF;

    FOREACH _table in ARRAY tables
    LOOP
      RAISE DEBUG '% delete in dup schema for hub_id % table %', clock_timestamp(), rec.hub_id, _table;
      EXECUTE 'DELETE FROM dup.' || _table || ' WHERE hostkey IN (SELECT * FROM deletes_' || rec.hub_id || ')';
      IF _table = '__lastseenhosts'::text THEN
        EXECUTE 'DELETE FROM dup.' || _table || ' WHERE remotehostkey IN (SELECT * FROM deletes_' || rec.hub_id || ')';
      END IF;
      RAISE DEBUG '% insert from % schema to dup for hub_id % table %', clock_timestamp(), source_schema, rec.hub_id, _table;
      EXECUTE 'INSERT INTO dup.' || _table || ' SELECT * FROM ' || source_schema || '.' || _table || ' WHERE hostkey in (SELECT * FROM duplicates_' || rec.hub_id || ')';
      RAISE DEBUG '% delete from % schema for hub_id % table %', clock_timestamp(), source_schema, rec.hub_id, _table;
      EXECUTE 'DELETE FROM ' || source_schema || '.' || _table || ' WHERE hostkey in (SELECT * FROM duplicates_' || rec.hub_id || ')';
    END LOOP;
  END LOOP;

  RAISE DEBUG 'Detaching importing tables from % schema', tmp_schema;
  FOREACH _table in ARRAY tables
  LOOP
    FOREACH feeder_hub_id in ARRAY importing_hub_ids
    LOOP
      EXECUTE 'ALTER TABLE ' || tmp_schema || '.' || _table || ' DETACH PARTITION hub_' || feeder_hub_id || '_importing.' || _table;
    END LOOP;
  END LOOP;

  RAISE DEBUG 'Deleting % schema', tmp_schema;
  EXECUTE 'DROP SCHEMA ' || tmp_schema || ' CASCADE';

  PERFORM public.release_exclusive_schema_lock('handle_duplicate_hostkeys_in_import');

  RAISE DEBUG '% Done handling duplicate hostkeys in import', clock_timestamp();
EXCEPTION
WHEN others then
  RAISE NOTICE 'exception during handle_duplicate_hostkeys_in_import(), cleaning up';

  IF verify_schema(tmp_schema) THEN
    EXECUTE 'DROP SCHEMA ' || tmp_schema || ' CASCADE';
  END IF;
  GET STACKED DIAGNOSTICS error_code = RETURNED_SQLSTATE,
                          error_message = MESSAGE_TEXT,
                          error_stack = PG_EXCEPTION_CONTEXT;

  PERFORM public.release_exclusive_schema_lock('exception in handle_duplicate_hostkeys_in_import');

  RAISE EXCEPTION 'handle_duplicate_hostkeys_in_import() got an exception: RETURNED_SQLSTATE: %, MESSAGE_TEXT: %, PG_EXCEPTION_CONTEXT: %', error_code, error_message, error_stack;
END;
$$ LANGUAGE plpgsql;

COMMENT ON FUNCTION handle_duplicate_hostkeys_in_import IS 'Given an array of hostkeys during Federated Reporting import, move duplicates based on __hosts.lastreporttimestamp to a dup schema not used by Mission Portal.';


CREATE OR REPLACE FUNCTION attach_feeder_schema(hostkey_search TEXT, table_whitelist TEXT[]) RETURNS VOID AS $$
DECLARE r RECORD;
        _tablename TEXT;
        orig_schema TEXT;
        new_schema TEXT;
        feeder_table TEXT;
        old_table TEXT;
        public_table TEXT;
        id NUMERIC;
        error_code TEXT;
        error_message TEXT;
        error_stack TEXT;
        indexes_cursor REFCURSOR;
        index_rec RECORD;
        index_sql TEXT;
BEGIN
  RAISE DEBUG 'ENTER attach_feeder_schema(%)', hostkey_search;
  id := get_hub_id(hostkey_search);
  orig_schema := 'hub_' || id;
  new_schema := 'hub_' || id || '_importing'; -- this is the schema in which the current import has been processed

  -- first create any needed indexes to speed up attaching tables later
  FOREACH _tablename in ARRAY table_whitelist
  LOOP
    OPEN indexes_cursor FOR SELECT indexdef FROM pg_indexes
      WHERE tablename = _tablename AND schemaname = 'public';
    LOOP
      FETCH indexes_cursor INTO index_rec;
      EXIT WHEN NOT FOUND;

      index_sql := index_rec.indexdef;
      index_sql := replace(index_sql, 'public', new_schema);
      EXECUTE index_sql;
    END LOOP;
    CLOSE indexes_cursor;
  END LOOP;

  PERFORM public.obtain_exclusive_schema_lock();
  FOREACH _tablename IN ARRAY table_whitelist
  LOOP
    IF (_tablename != '__promiselog_*') THEN
      feeder_table := quote_ident(new_schema) || '.' || quote_ident(_tablename);
      old_table := quote_ident(orig_schema) || '.' || quote_ident(_tablename);
      public_table := quote_ident('public') || '.' || quote_ident(_tablename);

      RAISE DEBUG 'feeder_table: %', feeder_table;
      RAISE DEBUG 'old_table: %', old_table;
      RAISE DEBUG 'public_table: %', public_table;

      IF verify_inheritance(public_table, old_table) THEN
        EXECUTE 'ALTER TABLE ' || public_table || ' DETACH PARTITION ' || old_table;
      END IF;

      RAISE DEBUG 'adding check constraint on hub_id to feeder table';
      EXECUTE 'ALTER TABLE ' || feeder_table || ' ADD CONSTRAINT hub_id_check CHECK ( hub_id = ' || id || ' )';

      RAISE DEBUG 'attaching new table % to public table % for values in %', feeder_table, public_table, id;
      EXECUTE 'ALTER TABLE ' || public_table || ' ATTACH PARTITION ' || feeder_table || ' FOR VALUES IN (' || id || ')';

      RAISE DEBUG 'removing temporary check constraint on hub_id on feeder table';
      EXECUTE 'ALTER TABLE ' || feeder_table || ' DROP CONSTRAINT hub_id_check';

    ELSE
      RAISE DEBUG 'handle __promiselog child tables ';
      FOR r in SELECT table_name
               FROM information_schema.tables
               WHERE table_name ~ '__promiselog_'
                 AND table_schema = new_schema
      LOOP
        feeder_table := quote_ident(new_schema) || '.' || quote_ident(r.table_name);
        RAISE DEBUG 'setting hub_id in table % to %', feeder_table, id;
        EXECUTE 'UPDATE ' || feeder_table || ' SET hub_id = ' || id;
        EXECUTE 'ALTER TABLE ' || feeder_table || ' NO INHERIT ' || new_schema || '.__promiselog';
        EXECUTE 'ALTER TABLE ' || feeder_table || ' INHERIT public.__promiselog';
      END LOOP;

      -- remove temporary feeder __promiselog table now that all child tables
      -- are re-attached to public.__promiselog.
      EXECUTE 'DROP TABLE ' || new_schema || '.__promiselog CASCADE';

    END IF;
  END LOOP;

  IF verify_schema(orig_schema) THEN
    RAISE DEBUG 'schema attachment success, remove orig schema: %', orig_schema;
    EXECUTE 'DROP SCHEMA ' || orig_schema || ' CASCADE';
  END IF;

  EXECUTE 'ALTER SCHEMA ' || new_schema || ' RENAME TO ' || orig_schema;

  PERFORM public.release_exclusive_schema_lock('attach_feeder_schema ' || hostkey_search);
EXCEPTION
WHEN others THEN
  RAISE NOTICE 'got exception during attach_feeder_schema(), cleaning up';

  GET STACKED DIAGNOSTICS error_code = RETURNED_SQLSTATE,
                          error_message = MESSAGE_TEXT,
                          error_stack = PG_EXCEPTION_CONTEXT;

  RAISE NOTICE 'attach_feeder_schema(%) got an exception: RETURNED_SQLSTATE: %, MESSAGE_TEXT: %, PG_EXCEPTION_CONTEXT: %', hostkey_search, error_code, error_message, error_stack;

  IF verify_schema(new_schema) THEN
    RAISE DEBUG 'Dropping new schema % which failed to complete', new_schema;
    EXECUTE 'DROP SCHEMA ' || new_schema || ' CASCADE';
  END IF;

  PERFORM public.release_exclusive_schema_lock('exception in attach_feeder_schema ' || hostkey_search);
  RAISE EXCEPTION 'attach_feeder_schema(%) got an exception: RETURNED_SQLSTATE: %, MESSAGE_TEXT: %, PG_EXCEPTION_CONTEXT: %', hostkey_search, error_code, error_message, error_stack;
END;
$$ LANGUAGE plpgsql;

-- superhub_schema() returns exit_code=<exit_code> for parsing by wrapper script
-- exit_code = 0 (kept), 1 (repaired), 2 (failed)
DROP FUNCTION IF EXISTS superhub_schema(); -- to allow changes to function signature
DROP FUNCTION IF EXISTS superhub_schema(TEXT);
CREATE OR REPLACE FUNCTION superhub_schema(superhub_hostkey TEXT) RETURNS TEXT AS $superhub_schema$
DECLARE r record;
        hub_table TEXT;
        public_table TEXT;
        pk_name TEXT;
        exit_code INT = 2; -- failed (not kept)
        error_message TEXT;
        error_stack TEXT;
        indexes_cursor REFCURSOR;
        index_rec RECORD;
        index_sql TEXT;
        skip_tables TEXT[] = ARRAY[
          '__hubs',
          '__hub',
          '__health_diagnostics_dismissed',
          'public.__Views',
          'variables_dictionary' -- because it is generated from partitioned __inventory
        ];

BEGIN
  IF NOT public.is_superhub() THEN
    BEGIN
      PERFORM public.obtain_exclusive_schema_lock();

      -- note that any changes to __hubs should be synchronized in mission-portal/tests/phpunit/models/RemoteHubModelTest.php
      CREATE TABLE __hubs (hub_id BIGSERIAL, hostkey TEXT PRIMARY KEY, last_import_ts TIMESTAMPTZ);
      INSERT INTO __hubs (hub_id, hostkey) VALUES (0, superhub_hostkey);

      CREATE SCHEMA hub_0;

      FOR r in SELECT table_name FROM information_schema.tables
               WHERE table_type = 'BASE TABLE' AND table_schema = 'public'
      LOOP
        hub_table := quote_ident('hub_0') || '.' || quote_ident(r.table_name);
        public_table := quote_ident(r.table_name);

        IF (NOT ARRAY[public_table] <@ skip_tables) THEN
          RAISE DEBUG 'REPAIRING table %', public_table;

          RAISE DEBUG 'adding hub_id to table %', public_table;
          EXECUTE 'ALTER TABLE ' || public_table || ' ADD COLUMN IF NOT EXISTS hub_id BIGINT DEFAULT 0';

          IF (public_table !~ '__promiselog*') THEN
            -- first get all indexes on the public table and keep them for later
            OPEN indexes_cursor FOR SELECT indexdef FROM pg_indexes
              WHERE tablename = public_table AND schemaname = 'public';

            -- must move the public table to hub_0 because tables can't be altered to be partitioned
            RAISE DEBUG 'moving table % from public to hub_0', public_table;
            EXECUTE 'ALTER TABLE ' || public_table || ' SET SCHEMA "hub_0"';

            RAISE DEBUG 'creating public partitioned table and attaching hub table to it';
            EXECUTE 'CREATE TABLE ' || public_table || ' (like ' || hub_table || ' INCLUDING DEFAULTS) PARTITION BY LIST (hub_id)';
            EXECUTE 'ALTER TABLE ' || public_table || ' ATTACH PARTITION ' || hub_table || ' FOR VALUES IN (0)';

            -- recreate the indexes on the public partitioned table with the addition of hub_id column
            -- since feeder schemas' tables will attach they will inherit all indexes on the public tables
            LOOP
              FETCH indexes_cursor INTO index_rec;
              EXIT WHEN NOT FOUND;

              index_sql := index_rec.indexdef;
              IF position('UNIQUE' in index_sql) != 0 THEN
                index_sql := regexp_replace(index_sql,'\((.*?)\)','(\1, hub_id)');
              END IF;

              index_sql := replace(index_sql,'hub_0','public');
              BEGIN
                EXECUTE index_sql;
              EXCEPTION
              WHEN others THEN
                RAISE WARNING 'failed to create index "%s"', index_sql;
              END;
            END LOOP;

            CLOSE indexes_cursor;
          END IF;
        END IF;
      END LOOP;

      EXECUTE 'SELECT public.ensure_views()';

      -- add hub_id column as part of primary key in __promiselog
      ALTER TABLE __PromiseLog DROP CONSTRAINT IF EXISTS __promiselog_pkey;
      ALTER TABLE __PromiseLog ADD PRIMARY KEY (id, hub_id);

      PERFORM public.release_exclusive_schema_lock('superhub_schema ' || superhub_hostkey);

      exit_code = 1; -- repaired

    EXCEPTION
      WHEN others THEN
        GET STACKED DIAGNOSTICS error_message = MESSAGE_TEXT,
                                error_stack = PG_EXCEPTION_CONTEXT;
        RAISE NOTICE E'schema modifications failed: %\n%', error_message, error_stack;

        PERFORM public.release_exclusive_schema_lock('exception in superhub_schema ' || superhub_hostkey);

        exit_code = 2; -- not kept
    END;
  ELSE -- __hubs table exists
    exit_code = 0; -- kept
  END IF;

  RETURN 'exit_code=' || exit_code;
END;
$superhub_schema$ LANGUAGE plpgsql;

-------------------------------------------------------------------------------

-- Virtual View
--CREATE TABLE IF NOT EXISTS Inventory (
--        HostKey             TEXT,
--        KeyName             TEXT,
--        Type                TEXT,
--        MetaTags            TEXT[],
--        Value               TEXT
--    );
CREATE OR REPLACE FUNCTION ensure_inventory_view() RETURNS VOID AS $body$
DECLARE hub_id_text TEXT;
BEGIN
  IF public.is_superhub() THEN
    hub_id_text = ', hub_id';
  END IF;

  EXECUTE format('
CREATE OR REPLACE VIEW Inventory WITH (security_barrier) AS (
    SELECT
        HostKey,
        KeyName,
        Type,
        MetaTags,
        Value
        %s
    FROM (
        SELECT
            HostKey,
            ContextName AS KeyName,
            ''context'' AS Type,
            MetaTags,
            ContextName AS Value
            %s
        FROM
            Contexts
        WHERE
            ''inventory'' = ANY (MetaTags)
            AND NOT ''attribute_name=none'' = ANY(MetaTags)
        UNION ALL
        SELECT
            HostKey,
            comp AS KeyName,
            VariableType AS Type,
            MetaTags,
            VariableValue AS Value
            %s
        FROM
            Variables
        WHERE
            ''inventory'' = ANY (MetaTags)
            AND NOT ''attribute_name=none'' = ANY(MetaTags)
    ) tmp
);
', hub_id_text, hub_id_text, hub_id_text);
END;
$body$ LANGUAGE plpgsql;
PERFORM public.ensure_view('FUNCTION', 'ensure_inventory_view');
COMMENT ON VIEW Inventory is 'This is old inventory view that contains fetched data from __variables and __contexts. This view is no longer used by API or Mission portal and exists only for backward compatibility';

---- ENT-4612 ensure provided feeders are added to __hubs table ----
-- prints to stderr exit_code=<exit_code> for parsing by wrapper script
-- exit_code = 0 (kept), 1 (repaired), 2 (failed)
-- @args feeders TEXT[]
DROP FUNCTION IF EXISTS ensure_feeders(); -- to allow changes to function signature
CREATE OR REPLACE FUNCTION ensure_feeders(hostkeys TEXT[]) RETURNS TEXT AS $ensure_hubs$
DECLARE
  hostkey TEXT;
  exit_code INT = 0;

BEGIN
  RAISE DEBUG 'hostkeys are %', hostkeys;
  FOREACH hostkey IN ARRAY hostkeys
  LOOP
    IF hostkey = '' THEN
      RAISE EXCEPTION 'hostkey cannot be empty';
    END IF;
    RAISE DEBUG 'make sure hostkey % is present in __hubs', hostkey;
    BEGIN
      INSERT INTO __hubs (hostkey) VALUES (hostkey);
      exit_code = 1;
    EXCEPTION
    WHEN unique_violation THEN
      RAISE DEBUG '% is already present in __hubs', hostkey;
    WHEN others THEN
      exit_code = 2;
    END;
  END LOOP;
  RETURN 'exit_code=' || exit_code;
END;
$ensure_hubs$ LANGUAGE plpgsql;

-- ENT-4862 Create view to select promises not kept (failed) that have not been kept or repaired
CREATE OR REPLACE FUNCTION ensure_not_kept_not_repaired_view() RETURNS VOID AS $body$
BEGIN
  EXECUTE 'CREATE OR REPLACE VIEW not_kept_not_repaired WITH (security_barrier) AS
    WITH p AS (
      SELECT * FROM __promiselog
      WHERE  promiseoutcome = ''NOTKEPT''
      AND NOT EXISTS (
        SELECT 1 FROM __promiseexecutions
        WHERE __promiselog.promisehash = __promiseexecutions.promisehash
        )
      )
      SELECT p.* FROM p WHERE  (
        CASE WHEN
          coalesce(current_setting(''rbac.filter'', true), '''') = ''''
        THEN
          p.hostkey IS NOT NULL
        ELSE
          p.hostkey IN (SELECT * FROM get_rbac_hostkeys())
        END );';
END;
$body$ LANGUAGE plpgsql;


PERFORM public.ensure_view('FUNCTION', 'ensure_not_kept_not_repaired_view');

COMMENT ON VIEW not_kept_not_repaired IS 'View to select promises not kept (failed) that have not been kept or repaired';

CREATE TABLE IF NOT EXISTS __health_diagnostics_failures (
  hostkey text NOT NULL,
  category text NOT NULL,
  discovered_at timestamptz NOT NULL DEFAULT now()
);
COMMENT ON TABLE __health_diagnostics_failures IS 'Stores health diagnostics failures.';

PERFORM public.ensure_view('RBAC', '__health_diagnostics_failures');


END IF; -- current_schema() = 'public'
END $PUBLIC$;

CREATE TABLE IF NOT EXISTS __cmdb (
  hostkey  text PRIMARY KEY,
  value  jsonb NOT NULL,
  updated_at timestamptz NOT NULL DEFAULT now(),
  epoch bigint NOT NULL DEFAULT 0
);

CREATE INDEX IF NOT EXISTS  cmdb_updated_at ON __cmdb (updated_at);
CREATE INDEX IF NOT EXISTS  cmdb_epoch ON __cmdb (epoch);
COMMENT ON TABLE __cmdb IS 'Stores hosts-specific CMDB data';
COMMENT ON COLUMN __cmdb.value IS 'JSON column with a structure {classes: [json objects], variables: [json objects]}';
COMMENT ON COLUMN __cmdb.epoch IS 'Contains sequence number of the latest cmdb change';

DO $PUBLIC$
BEGIN
  IF current_schema() = 'public' THEN

PERFORM public.ensure_view('RBAC', '__cmdb');

CREATE OR REPLACE FUNCTION update_cmdb_data_file()
RETURNS TRIGGER AS $$
BEGIN
  EXECUTE 'NOTIFY cmdb_refresh, ''' || NEW.hostkey || '''';
  RETURN NULL; -- result is ignored since this is an AFTER trigger
END;
$$ LANGUAGE plpgsql;

COMMENT ON FUNCTION update_cmdb_data_file is 'A trigger procedure that notifies cf-reactor to update CMDB data file when changed in the DB';

CREATE OR REPLACE FUNCTION delete_cmdb_data_file()
RETURNS TRIGGER AS $$
BEGIN
  EXECUTE 'NOTIFY cmdb_refresh, ''' || OLD.hostkey || '''';
  RETURN NULL; -- result is ignored since this is an AFTER trigger
END;
$$ LANGUAGE plpgsql;

COMMENT ON FUNCTION delete_cmdb_data_file is 'A trigger procedure that notifies cf-reactor to delete CMDB data file when data deleted in the DB';

END IF; -- public schema
END $PUBLIC$;

-- CREATE OR REPLACE TRIGGER only works on PostgreSQL 14+
DO $$
BEGIN
  -- always drop the public trigger since in the case of superhub and hub_0 we want that gone.
  DROP TRIGGER IF EXISTS update_cmdb_data_file on public.__cmdb;
  DROP TRIGGER IF EXISTS delete_cmdb_data_file on public.__cmdb;
  IF (public.is_superhub() AND current_schema() = 'hub_0') OR NOT public.is_superhub() THEN
    DROP TRIGGER IF EXISTS update_cmdb_data_file ON __cmdb;
    CREATE TRIGGER update_cmdb_data_file
      AFTER INSERT OR UPDATE ON __cmdb
      FOR EACH ROW EXECUTE FUNCTION public.update_cmdb_data_file();

    DROP TRIGGER IF EXISTS delete_cmdb_data_file ON __cmdb;
    CREATE TRIGGER delete_cmdb_data_file
      AFTER DELETE ON __cmdb
      FOR EACH ROW EXECUTE FUNCTION public.delete_cmdb_data_file();
  END IF;
END$$;

-- ENT-7522 Fix spelling error in 'received'
UPDATE diagnostics set name = 'received_data_size_per_host' where name = 'recivied_data_size_per_host';

SELECT public.ensure_view('RBAC', 'contextcache', 'v_contextcache');

-- this schema will own shared_host_groups fdw table and function to work with groups
CREATE SCHEMA IF NOT EXISTS groups;
-- create foreign data wrapper extension needed to call cfsettings database from cfdb
CREATE EXTENSION IF NOT EXISTS postgres_fdw;
DROP SERVER IF EXISTS cfsettings_fdw CASCADE;
CREATE SERVER cfsettings_fdw FOREIGN DATA WRAPPER postgres_fdw OPTIONS (dbname 'cfsettings');
DROP USER MAPPING IF EXISTS  FOR cfpostgres SERVER server_name;
CREATE USER MAPPING FOR cfpostgres
    SERVER cfsettings_fdw
    OPTIONS (user 'cfpostgres');
CREATE USER MAPPING FOR root
    SERVER cfsettings_fdw
    OPTIONS (user 'root');
CREATE USER MAPPING FOR cfapache
    SERVER cfsettings_fdw
    OPTIONS (user 'cfapache');
GRANT USAGE ON FOREIGN SERVER cfsettings_fdw TO cfpostgres;
GRANT USAGE ON FOREIGN SERVER cfsettings_fdw TO root;
GRANT USAGE ON FOREIGN SERVER cfsettings_fdw TO cfapache;

IMPORT FOREIGN SCHEMA public LIMIT TO (
    shared_host_groups,
    personal_host_groups,
    shared_host_groups_data
    ) FROM SERVER cfsettings_fdw INTO groups;

CREATE OR REPLACE FUNCTION groups.get_hosts_in_shared_group(group_id_value INTEGER)
    RETURNS TABLE(hostkey TEXT) AS $$
DECLARE
    filter_sql TEXT;
BEGIN
    SET transaction_read_only = on;
    SET search_path TO groups,public;
    SELECT shared_host_groups.filter_sql INTO filter_sql FROM shared_host_groups WHERE id = group_id_value AND deletion_time IS NULL;
    IF filter_sql IS NULL THEN
        RAISE EXCEPTION 'Shared group does not exist or the SQL representation of filters is empty.';
    END IF;

    IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'hosts_of_group_tmp') THEN
        RAISE DEBUG 'get_hosts_in_shared_group is using cached results.';
        RETURN QUERY SELECT unnest(hosts_of_group_tmp.hostkey) AS hostkey FROM hosts_of_group_tmp WHERE group_id = group_id_value;
    ELSE
       RAISE DEBUG 'get_hosts_in_shared_group function directly executing groups sql statements without cache applied.';
       -- filter_sql might query other tables/views, for example v_contextcache for classes filter
       -- as long as user cannot modify filter_sql directly via API and it's created from the filter_json automatically
       -- it's safe to use it here
       RETURN QUERY EXECUTE 'SELECT hostkey FROM inventory_new WHERE ' || filter_sql;
    END IF;
END $$ LANGUAGE 'plpgsql';

COMMENT ON FUNCTION groups.get_hosts_in_shared_group IS 'Returns all hosts (identified by their hostkeys) in the particular shared group.';

CREATE OR REPLACE FUNCTION groups.get_hosts_in_personal_group(group_id_value INTEGER, user_name TEXT)
    RETURNS TABLE(hostkey TEXT) AS $$
DECLARE
    filter_sql TEXT;
    tmpTableName TEXT = 'hosts_of_personal_group_' || user_name || '_tmp';
BEGIN
    -- allowed read only transactions for additional security reasons
    -- even if filter_sql cannot be modified directly via the API
    SET transaction_read_only = on;
    SET search_path TO groups,public;
    SELECT personal_host_groups.filter_sql INTO filter_sql FROM personal_host_groups WHERE id = group_id_value AND owner = user_name;
    IF filter_sql IS NULL THEN
        RAISE EXCEPTION 'Personal group #% does not exist or the SQL representation of filters is empty. Username: %', group_id_value, user_name;
    END IF;

    IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = tmpTableName) THEN
        RAISE DEBUG 'get_hosts_in_personal_group(%, %) is using cached results.', group_id_value, user_name;
        -- use session cache if the temporary table exists
        -- this performs multiple sequences of queries, especially when you get groups that belong to hosts.
        RETURN QUERY EXECUTE 'SELECT unnest('|| tmpTableName ||'.hostkey) AS hostkey FROM '|| tmpTableName ||' WHERE group_id = group_id_value';
    ELSE
        RAISE DEBUG 'get_hosts_in_personal_group(%, %) function directly executing groups sql statements without cache applied.', group_id_value, user_name;
        -- filter_sql might query other tables/views, for example v_contextcache for classes filter
        -- as long as user cannot modify filter_sql directly via API and it's created from the filter_json automatically
        -- it's safe to use it here
        RETURN QUERY EXECUTE 'SELECT hostkey FROM inventory_new WHERE ' || filter_sql;
    END IF;
END $$ LANGUAGE 'plpgsql';

COMMENT ON FUNCTION groups.get_hosts_in_personal_group IS 'Returns all hosts (identified by their hostkeys) in the particular personal group.';


CREATE OR REPLACE FUNCTION groups.get_shared_groups_of_host(hostkey_value TEXT)
    RETURNS TABLE(id INTEGER) AS $$
DECLARE
    group_rec RECORD;
    groups_ids INTEGER[] := '{}';
    alias TEXT;
    resulted_sql TEXT;
    resulted_select_array  TEXT[] := '{}';
    resulted_join TEXT := '';
BEGIN
    SET search_path TO groups,public;
    SET transaction_read_only = on;
    IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'groups_of_host_tmp') THEN
        RAISE DEBUG 'get_shared_groups_of_host is using cached results.';
        RETURN QUERY SELECT unnest(group_id) as group_id FROM groups_of_host_tmp WHERE hostkey = hostkey_value;
    ELSE
        RAISE DEBUG 'get_shared_groups_of_host function directly executing groups sql statements without cache applied.';
        FOR group_rec IN SELECT shared_host_groups.id,  shared_host_groups.filter_sql FROM shared_host_groups WHERE deletion_time IS NULL
            LOOP
                -- we left join groups sql subquery and for this reason we need to set a subquery alias
                alias := 'group_' || group_rec.id;
                resulted_select_array := ARRAY_APPEND(resulted_select_array, 'count('|| alias ||'.*)');
                resulted_join := resulted_join || ' LEFT JOIN (SELECT hostkey FROM inventory_new WHERE '|| group_rec.filter_sql ||') '|| alias ||' ON '|| alias ||'.hostkey = inventory_new.hostkey ';
                groups_ids := ARRAY_APPEND(groups_ids, group_rec.id);
            END LOOP;

        IF array_length(groups_ids, 1) > 0 THEN
            -- resulted selects groups id which has hosts count more than 0
            resulted_sql := 'SELECT group_id FROM (' ||
                            'SELECT * FROM unnest( ' ||
                            'ARRAY[' || array_to_string(groups_ids, ',') || '], ' ||
                            '(SELECT ARRAY[' || array_to_string(resulted_select_array, ',') || '] ' ||
                            'FROM public.inventory_new '|| resulted_join ||' ' ||
                            'WHERE public.inventory_new.hostkey = ' || quote_nullable(hostkey_value)  || ' )' ||
                            ') AS data(group_id, hosts_count )' ||
                            ')subquery WHERE hosts_count > 0';
            RETURN query EXECUTE resulted_sql;
        ELSE
            -- Return an empty result set when there are no shared host groups
            RETURN;
        END IF;
    END IF;
END $$ LANGUAGE 'plpgsql';

CREATE OR REPLACE FUNCTION groups.is_host_in_shared_group(hostkey_value TEXT, group_id INTEGER)
    RETURNS BOOLEAN AS $$
DECLARE
    is_host_in_group BOOLEAN := false;
BEGIN
    SELECT true INTO is_host_in_group FROM groups.get_hosts_in_shared_group(group_id) t WHERE t.hostkey = hostkey_value;
    RETURN is_host_in_group;
END $$ LANGUAGE 'plpgsql';

COMMENT ON FUNCTION groups.is_host_in_shared_group IS 'Checking if the given host is a member of the given group.';

CREATE OR REPLACE PROCEDURE groups.create_shared_groups_hosts_cache(refresh BOOLEAN = false) AS $$
DECLARE
    group_rec RECORD;
BEGIN
    SET search_path TO groups;

    IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'groups_of_host_tmp') THEN
       IF refresh THEN
           CALL drop_shared_groups_hosts_cache();
       ELSE
           RETURN;
       END IF;
    END IF;

    CREATE TEMP TABLE IF NOT EXISTS groups_of_host_tmp (
      hostkey TEXT PRIMARY KEY,
      group_id INTEGER[]
    );
    CREATE TEMP TABLE IF NOT EXISTS hosts_of_group_tmp (
      hostkey TEXT[],
      group_id INTEGER PRIMARY KEY
    );
    SET search_path TO groups,public;
    SET transaction_read_only = on;
    FOR group_rec IN SELECT shared_host_groups.id,  shared_host_groups.filter_sql  FROM shared_host_groups WHERE deletion_time IS NULL
        LOOP
            EXECUTE 'INSERT INTO groups_of_host_tmp
                          (SELECT  hostkey,  ''{'|| group_rec.id ||'}'' as group_id FROM public.inventory_new WHERE  '|| group_rec.filter_sql ||')
                          ON CONFLICT (hostkey) DO UPDATE set group_id = groups_of_host_tmp.group_id || EXCLUDED.group_id';

            EXECUTE 'INSERT INTO hosts_of_group_tmp
                          (SELECT  array_agg(hostkey) as hostkey,  '|| group_rec.id ||' as group_id FROM public.inventory_new WHERE  '|| group_rec.filter_sql ||')
                          ON CONFLICT (group_id) DO UPDATE set hostkey = EXCLUDED.hostkey';
    END LOOP;
END $$ LANGUAGE 'plpgsql';

COMMENT ON PROCEDURE groups.create_shared_groups_hosts_cache(refresh BOOLEAN)
IS 'Create groups_of_host_tmp & hosts_of_group_tmp temporary tables. These table will last until the session end.';

CREATE OR REPLACE PROCEDURE groups.drop_shared_groups_hosts_cache() AS $$
BEGIN
    SET search_path TO groups;
    DROP TABLE IF EXISTS groups_of_host_tmp ;
    DROP TABLE IF EXISTS hosts_of_group_tmp ;
END $$ LANGUAGE 'plpgsql';

COMMENT ON PROCEDURE groups.drop_shared_groups_hosts_cache
    IS 'Drop groups_of_host_tmp & hosts_of_group_tmp temporary tables.';

CREATE OR REPLACE PROCEDURE groups.drop_personal_groups_hosts_cache(user_name TEXT) AS $$
BEGIN
    SET search_path TO groups;
    EXECUTE 'DROP TABLE IF EXISTS hosts_of_personal_group_' || user_name || '_tmp';
END $$ LANGUAGE 'plpgsql';

COMMENT ON PROCEDURE groups.drop_personal_groups_hosts_cache
    IS 'Drop hosts_of_personal_group_tmp temporary tables.';


CREATE OR REPLACE PROCEDURE groups.create_personal_groups_hosts_cache(user_name TEXT) AS $$
DECLARE
    group_rec RECORD;
    tableName TEXT = 'hosts_of_personal_group_' || user_name || '_tmp';
BEGIN
    SET search_path TO groups;
    CALL drop_personal_groups_hosts_cache(user_name);
    EXECUTE 'CREATE TEMP TABLE IF NOT EXISTS ' || tableName || ' (
     hostkey TEXT[],
     group_id INTEGER PRIMARY KEY
    )';
    SET search_path TO groups,public;
    SET transaction_read_only = on;
    FOR group_rec IN SELECT personal_host_groups.id,  personal_host_groups.filter_sql  FROM personal_host_groups WHERE personal_host_groups.owner = user_name
        LOOP
            EXECUTE 'INSERT INTO ' || tableName ||'
                     (SELECT  array_agg(hostkey) as hostkey,  '|| group_rec.id ||' as group_id FROM public.inventory_new WHERE  '|| group_rec.filter_sql ||')
                     ON CONFLICT (group_id) DO UPDATE set hostkey = EXCLUDED.hostkey';
        END LOOP;
END $$ LANGUAGE 'plpgsql';

COMMENT ON PROCEDURE groups.create_personal_groups_hosts_cache
    IS 'Create hosts_of_personal_group_tmp temporary table. This table will last until the session end.';

-- Deny all functions execution created in the public schema
REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM public;
-- Allowed functions list
GRANT EXECUTE ON FUNCTION public.get_rbac_hostkeys() TO public;
GRANT EXECUTE ON FUNCTION public.cf_clearSlist(varchar) TO public;
GRANT EXECUTE ON FUNCTION public.cf_convertToNum(text) TO public;
GRANT EXECUTE ON FUNCTION public.release_shared_schema_lock(TEXT) TO public;
GRANT EXECUTE ON FUNCTION public.release_schema_lock(BOOLEAN, TEXT) TO public;
GRANT EXECUTE ON FUNCTION public.obtain_shared_schema_lock() TO public;
GRANT EXECUTE ON FUNCTION public.obtain_schema_lock(BOOLEAN) TO public;
GRANT EXECUTE ON FUNCTION public.clear_hosts_references(TEXT) TO public;
GRANT EXECUTE ON FUNCTION public.gethostkeyandvaluesforattribute(TEXT) TO public;
GRANT EXECUTE ON FUNCTION public.gethostkeyandvaluesforkeyname(TEXT) TO public;
GRANT EXECUTE ON FUNCTION public.process_host_events() TO public;
GRANT EXECUTE ON FUNCTION public.release_exclusive_schema_lock(TEXT) TO public;
GRANT EXECUTE ON PROCEDURE public.upsert_host_with_ip(TEXT, TEXT) TO public;
GRANT EXECUTE ON FUNCTION public.reverse_lower_like(TEXT, TEXT) TO public;
GRANT EXECUTE ON FUNCTION public.reverse_lower_eq(TEXT, TEXT) TO public;

DO $PUBLIC$
    BEGIN
        IF current_schema() = 'public' THEN
            CREATE OR REPLACE FUNCTION ensure_deleted_hosts_report_view() RETURNS VOID AS $body$
            BEGIN
                CREATE OR REPLACE VIEW deleted_hosts_report WITH (security_barrier) AS
                SELECT __hosts.hostkey, __hosts.deleted as "Host deleted at", max(__hubconnectionerrors.checktimestamp) as "Last report attempt", ipaddress
                FROM __hosts
                         LEFT JOIN __hubconnectionerrors ON __hubconnectionerrors.hostkey = __hosts.hostkey
                WHERE __hosts.deleted IS NOT NULL AND   __hubconnectionerrors.checktimestamp > __hosts.deleted
                GROUP BY __hosts.hostkey, __hosts.deleted, ipaddress;
            END;
            $body$ LANGUAGE plpgsql;

            PERFORM public.ensure_view('FUNCTION', 'ensure_deleted_hosts_report_view');
            COMMENT ON VIEW deleted_hosts_report IS 'View to select removed hosts that still report';
        END IF;
    END $PUBLIC$;

DO $$
BEGIN
  IF current_schema() = 'public' THEN
    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'configuration_type') THEN
    CREATE TYPE configuration_type AS ENUM ('class', 'variable', 'inventory', 'policy_configuration', 'module_input');
    END IF;

    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'cmdb_item_type') THEN
    CREATE TYPE cmdb_item_type AS ENUM ('class', 'variable');
    END IF;
  END IF;  
END
$$;

CREATE TABLE IF NOT EXISTS __cmdb_host_entries (
    id BIGSERIAL PRIMARY KEY,
    hostkey TEXT NOT NULL,
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    name TEXT,
    description TEXT,
    tags TEXT[],
    type configuration_type NOT NULL,
    meta JSONB DEFAULT '{}'
);

CREATE TABLE IF NOT EXISTS __cmdb_host_subentries (
    id BIGSERIAL PRIMARY KEY,
    hostkey TEXT NOT NULL,
    entry_id BIGINT NOT NULL REFERENCES __cmdb_host_entries(id) ON DELETE CASCADE,
    item_name VARCHAR(255) NOT NULL,
    item_value JSONB DEFAULT NULL,
    item_type cmdb_item_type NOT NULL,
    CONSTRAINT __cmdb_host_subentries_unique UNIQUE (hostkey, item_name, item_type)
);

CREATE INDEX IF NOT EXISTS idx_cmdb_items_hostkey ON __cmdb_host_entries(hostkey);

DO $$
BEGIN
IF current_schema() = 'public' THEN

COMMENT ON TABLE __cmdb_host_entries IS 'Stores CMDB configurations for hosts.';
COMMENT ON COLUMN __cmdb_host_entries.id IS 'Unique identifier for each CMDB entry.';
COMMENT ON COLUMN __cmdb_host_entries.hostkey IS 'Unique host identifier. All tables can be joined by HostKey to connect data concerning same hosts.';
COMMENT ON COLUMN __cmdb_host_entries.created_at IS 'Timestamp when this CMDB entry was created.';
COMMENT ON COLUMN __cmdb_host_entries.name IS 'Name of the entry item.';
COMMENT ON COLUMN __cmdb_host_entries.description IS 'Description of the entry item.';
COMMENT ON COLUMN __cmdb_host_entries.tags IS 'Tags of the entry item.';
COMMENT ON COLUMN __cmdb_host_entries.type IS 'Type of the entry item. Allowed values: inventory, variable, class, policy_configuration';
COMMENT ON COLUMN __cmdb_host_entries.meta IS 'Meta data of the entry item.';

COMMENT ON TABLE __cmdb_host_subentries IS 'Stores individual data elements for each entry item. Implements one-to-many relationship where each CMDB entry can have multiple subentries.';
COMMENT ON COLUMN __cmdb_host_subentries.id IS 'Unique identifier for each data element.';
COMMENT ON COLUMN __cmdb_host_subentries.hostkey IS 'Unique host identifier. All tables can be joined by HostKey to connect data concerning same hosts.';
COMMENT ON COLUMN __cmdb_host_subentries.entry_id IS 'Foreign key reference to __cmdb_host_entries.id.';
COMMENT ON COLUMN __cmdb_host_subentries.item_name IS 'Name of the data. Can be class/variable name';
COMMENT ON COLUMN __cmdb_host_subentries.item_value IS 'Value of the subentry data item as JSONB type.';
COMMENT ON COLUMN __cmdb_host_subentries.item_type IS 'CMDB data type, can be variable or class.';


CREATE OR REPLACE FUNCTION ensure_cmdb_host_entries_view() RETURNS VOID AS $body$
BEGIN
  DROP VIEW IF EXISTS cmdb_host_entries;
  CREATE VIEW cmdb_host_entries WITH (security_barrier) AS

  SELECT p.*,
         (SELECT json_agg(__cmdb_host_subentries) FROM __cmdb_host_subentries WHERE __cmdb_host_subentries.entry_id = p.id) AS entries
FROM __cmdb_host_entries AS p
WHERE  (
    CASE WHEN
      coalesce(current_setting('rbac.filter', true), '') = ''
    THEN
      p.hostkey IS NOT NULL
    ELSE
      p.hostkey IN (SELECT * FROM get_rbac_hostkeys())
    END );
END;
$body$ LANGUAGE plpgsql;

PERFORM public.ensure_view('FUNCTION', 'ensure_cmdb_host_entries_view');
PERFORM public.ensure_view('RBAC', '__cmdb_host_subentries', 'cmdb_host_subentries');

END IF; -- if current_schema() = 'public'
END $$;

-- Select 
CREATE OR REPLACE FUNCTION update_rendered_cmdb_data()
RETURNS TRIGGER AS $$
DECLARE
    affected_hostkey TEXT;
    classes_json JSONB := '{}';
    variables_json JSONB := '{}';
    final_json JSONB;
    current_epoch BIGINT;
BEGIN
    -- Identify affected hostkey
    -- For delete rows the hostkey is in the OLD and inserted/updated in the NEW
    IF TG_OP = 'DELETE' THEN
        affected_hostkey := OLD.hostkey;
    ELSE
        affected_hostkey := NEW.hostkey;
    END IF;
    
    -- Get current epoch and increment it
    SELECT COALESCE(MAX(epoch), 0) + 1 INTO current_epoch FROM __cmdb;
    
    -- Build classes JSON and store in declared classes_json variable
    -- skip value for classes
    SELECT COALESCE(
        jsonb_object_agg(
            s.item_name,
            jsonb_build_object(
                'tags', CASE 
                        WHEN e.tags IS NOT NULL THEN to_jsonb(e.tags)
                        ELSE '[]'::jsonb
                    END,
                'comment', COALESCE(e.description, '')
             )
        ), '{}'::jsonb
    ) INTO classes_json
    FROM __cmdb_host_subentries s
    JOIN __cmdb_host_entries e ON s.entry_id = e.id
    WHERE s.hostkey = affected_hostkey 
    AND s.item_type = 'class';
    
    -- Build variables JSON and store in declared variables_json variable
    SELECT COALESCE(
        jsonb_object_agg(
            s.item_name,
            jsonb_build_object(
                'tags', CASE 
                        WHEN e.tags IS NOT NULL THEN to_jsonb(e.tags)
                        ELSE '[]'::jsonb
                    END,
                'value', COALESCE(s.item_value, 'null'::jsonb),
                'comment', COALESCE(e.description, '')
            )
        ), '{}'::jsonb
    ) INTO variables_json
    FROM __cmdb_host_subentries s
    JOIN __cmdb_host_entries e ON s.entry_id = e.id
    WHERE s.hostkey = affected_hostkey 
    AND s.item_type = 'variable';
    
    -- Combine into final JSON structure
    final_json := jsonb_build_object(
        'classes', classes_json,
        'variables', variables_json
    );

    INSERT INTO __cmdb (hostkey, value, updated_at, epoch)
    VALUES (affected_hostkey, final_json, NOW(), current_epoch)
    ON CONFLICT (hostkey) 
    DO UPDATE SET 
        value = EXCLUDED.value,
        updated_at = EXCLUDED.updated_at,
        epoch = EXCLUDED.epoch;

    RETURN NULL;    
END;
$$ LANGUAGE plpgsql;

DO $$
BEGIN
IF current_schema() = 'public' AND NOT public.is_superhub() THEN
  DROP TRIGGER IF EXISTS cmdb_update_final_json_trigger ON public.__cmdb_host_subentries;
  -- when rows inside __cmdb_host_subentries are deleted/updated or inserted we
  -- call update_rendered_cmdb_data that will render the final json and update __cmdb table.
  -- cf-reactor on the __cmdb update will create host-specific JSON file
  CREATE TRIGGER cmdb_update_final_json_trigger
      AFTER INSERT OR UPDATE OR DELETE ON public.__cmdb_host_subentries
      FOR EACH ROW
      EXECUTE FUNCTION public.update_rendered_cmdb_data();
END IF;
END $$;
