# shellcheck shell=bash
# only allow this script to be sourced, by cf-support.sh, and not run directly
# as it relies on functions defined in cf-support.sh

# Here we use a bash quirk which allows return to be called outside of a function in a sourced file
# and a subshell to protect from returning from the script environment.
# If executed directly calling return will result in an error and return
# non-zero so proceed to the else near the end of this file..
if ! (return 0 2>/dev/null); then
  echo "run /var/cfengine/bin/cf-support instead of this file"
  exit 1
fi

# check if we are indeed on a hub
# this script should only be packaged in hub packages but just to be sure
# let's do some checks before-hand
if [ ! -d /var/cfengine/httpd ]; then
  echo "It seems that this host is not a CFEngine Enterprise Hub. Exiting."
  exit 1
fi

if ps -efl | grep cf-serve[rd]; then
  log_cmd "cf-serverd -Fv | awk '/=== BEGIN summary of access promises ===/,/=== END summary of access promises ===/'"
else
  echo "Could not get summary of access promises. cf-serverd must be running and it was not."
fi

# Collect all the PostgreSQL log files
for f in /var/log/postgresql.log*; do
    gzip_add "$f"
done

# tmpdir defined in cf-support which sources this file so disable check
# shellcheck disable=SC2154
mkdir -p "$tmpdir$WORKDIR/httpd"
"$BINDIR"/rsync -az "$WORKDIR/httpd/logs" "$tmpdir$WORKDIR/httpd/"

file_add "$WORKDIR"/state/pg/data/postgresql.conf
file_add "$WORKDIR"/state/pg/data/pg_hba.conf

# Collect settings from the running PostgreSQL instance
"$BINDIR/psql" cfdb -c "\copy (SELECT * from pg_settings) To '$tmpdir/pg_settings.csv' With CSV DELIMITER ',' HEADER"

# Hostkeys in one DB but not in another
LASTSEEN_LMDB_FILE="/var/cfengine/state/cf_lastseen.lmdb"
BINDIR="/var/cfengine/bin"
# Temp files for different sources
lmdb_hostkeys_a_temp=$(mktemp)
lmdb_hostkeys_k_temp=$(mktemp)
lmdb_hostkeys_qi_temp=$(mktemp)
lmdb_hostkeys_qo_temp=$(mktemp)
lmdb_combined_hostkeys_temp=$(mktemp)
postgres_hostkeys_temp=$(mktemp)
# Get hostkeys from lmdb
cf-check dump "$LASTSEEN_LMDB_FILE" | grep '"SHA=' | awk -F'"' '{print $4}' > $lmdb_hostkeys_a_temp
cf-check dump "$LASTSEEN_LMDB_FILE" | grep '"kSHA=' | awk -F'"' '{print $2}' | sed 's/^k//' > $lmdb_hostkeys_k_temp
cf-check dump "$LASTSEEN_LMDB_FILE" | grep '"qiSHA=' | awk -F'"' '{print $2}' | sed 's/^qi//' > $lmdb_hostkeys_qi_temp
cf-check dump "$LASTSEEN_LMDB_FILE" | grep '"qoSHA=' | awk -F'"' '{print $2}' | sed 's/^qo//' > $lmdb_hostkeys_qo_temp
cat $lmdb_hostkeys_a_temp $lmdb_hostkeys_k_temp $lmdb_hostkeys_qi_temp $lmdb_hostkeys_qo_temp | sort | uniq > $lmdb_combined_hostkeys_temp
# Get hostkeys from PostgreSQL
"$BINDIR/psql" cfdb -t -c "SELECT hostkey FROM __hosts;" | sed 's/^ //g' | sort | uniq > $postgres_hostkeys_temp
sed -i '/^$/d' $postgres_hostkeys_temp
comm -23 "$lmdb_combined_hostkeys_temp" "$postgres_hostkeys_temp" > $tmpdir/hostkeys-in-lmdb-not-pg.txt
comm -13 "$lmdb_combined_hostkeys_temp" "$postgres_hostkeys_temp" > $tmpdir/hostkeys-in-pg-not-lmdb.txt
# Clean up temporary files
rm "$lmdb_hostkeys_a_temp" "$lmdb_hostkeys_k_temp" "$lmdb_hostkeys_qi_temp" "$lmdb_hostkeys_qo_temp" "$lmdb_combined_hostkeys_temp" "$postgres_hostkeys_temp"
echo "** Host keys in one DB but not another" >> $tmpdir/system-info.txt
echo "- $(cat $tmpdir/hostkeys-in-lmdb-not-pg.txt | wc -l) hostkeys in lastseen LMDB not in PostgreSQL __hosts [[./hostkeys-in-lmdb-not-pg.txt]]" >> $tmpdir/system-info.txt
echo "- $(cat $tmpdir/hostkeys-in-pg-not-lmdb.txt | wc -l) hostkeys in PostgreSQL __hosts not in lastseen LMDB [[./hostkeys-in-pg-not-lmdb.txt]]" >> $tmpdir/system-info.txt

# Two machines with one hostkey share the hub's single collection cursor.
"$BINDIR/psql" cfdb -c "SELECT hostkey, hostname, hostkeycollisions, ipaddress, lastreporttimestamp
  FROM hosts
  WHERE hostkeycollisions > 0
  ORDER BY hostkeycollisions DESC
  LIMIT 100;" > "$tmpdir"/psql-hostkey-collisions.log 2>&1
_collision_hosts=$("$BINDIR/psql" cfdb -t -c "SELECT count(*) FROM hosts WHERE hostkeycollisions > 0;" 2>/dev/null | tr -d ' ')
echo "** Possible duplicate hosts (same hostkey, different machines)" >> "$tmpdir"/system-info.txt
echo "- ${_collision_hosts:-unknown} hosts with HostkeyCollisions > 0 [[./psql-hostkey-collisions.log]]" >> "$tmpdir"/system-info.txt

# Ordered __status history for hosts that are not CONSUMED, 50 rows per host.
"$BINDIR/psql" cfdb -c "SELECT host, ts, hubts, status, type
  FROM (
    SELECT host, ts, hubts, status, type,
           row_number() OVER (PARTITION BY host ORDER BY hubts DESC) AS rn
    FROM __status
    WHERE host IN (SELECT DISTINCT host FROM __status WHERE status <> 'CONSUMED')
  ) recent
  WHERE rn <= 50
  ORDER BY host, hubts DESC;" > "$tmpdir"/psql-status-history-failing-hosts.log 2>&1

# Why the hub could not reach or read from a client.
"$BINDIR/psql" cfdb -c "SELECT hostkey, querytype, count(*) AS errors,
    min(checktimestamp) AS earliest, max(checktimestamp) AS latest
  FROM __hubconnectionerrors
  GROUP BY hostkey, querytype
  ORDER BY errors DESC
  LIMIT 100;" > "$tmpdir"/psql-hub-connection-errors.log 2>&1

# Inventory attributes near the 1024-byte variable value cap.
"$BINDIR/psql" cfdb -c "SELECT substring(metatags::text from 'attribute_name=[\"]?([^\"},]+)') AS attribute,
    count(*) AS host_rows,
    max(length(variablename)) AS max_name_len,
    max(length(coalesce(variablevalue,''))) AS max_value_len,
    count(*) FILTER (WHERE length(coalesce(variablevalue,'')) = 1023) AS truncated,
    count(*) FILTER (WHERE length(coalesce(variablevalue,'')) = 1024) AS at_boundary
  FROM __variables
  WHERE metatags::text LIKE '%attribute_name%'
  GROUP BY 1 ORDER BY max_value_len DESC NULLS LAST LIMIT 100;" > "$tmpdir"/psql-inventory-attribute-shape.log 2>&1

# Variable values at the CF_MAXVARSIZE cap, only logged at VERBOSE otherwise.
"$BINDIR/psql" cfdb -c "SELECT count(*) AS values_at_cap,
    count(DISTINCT hostkey) AS hosts_affected
  FROM __variables
  WHERE length(VariableValue) >= 1023;" > "$tmpdir"/psql-variables-truncated.log 2>&1
"$BINDIR/psql" cfdb -c "SELECT namespace, bundle, variablename,
    length(VariableValue) AS len,
    CASE length(VariableValue) WHEN 1023 THEN 'truncated - reports a stale value'
                               WHEN 1024 THEN 'at the boundary, not truncated'
                               ELSE 'other' END AS state,
    count(*) AS hosts
  FROM __variables
  WHERE length(VariableValue) >= 1023
  GROUP BY namespace, bundle, variablename, len, state
  ORDER BY hosts DESC
  LIMIT 100;" >> "$tmpdir"/psql-variables-truncated.log 2>&1
_truncated_vars=$("$BINDIR/psql" cfdb -t -c "SELECT count(*) FROM __variables WHERE length(VariableValue) = 1023;" 2>/dev/null | tr -d ' ')
_boundary_vars=$("$BINDIR/psql" cfdb -t -c "SELECT count(*) FROM __variables WHERE length(VariableValue) = 1024;" 2>/dev/null | tr -d ' ')
{
  echo "** Variable values at the CF_MAXVARSIZE truncation cap"
  echo "- ${_truncated_vars:-unknown} values are exactly 1023 bytes, i.e. truncated and reporting a stale value [[./psql-variables-truncated.log]]"
  echo "- ${_boundary_vars:-unknown} values are exactly 1024 bytes, i.e. one byte from truncating"
} >> "$tmpdir"/system-info.txt

# Per-batch collection summary, only written when cf-hub runs with -l.
for _hub_log in "$WORKDIR/hub_log" /var/log/cfengine/hub_log; do
  if [ -f "$_hub_log" ]; then
    gzip_add "$_hub_log"
    break
  fi
done

# Show 20 largest tables in cfdb database
# see https://www.postgresql.org/docs/current/catalog-pg-class.html
# and https://www.postgresql.org/docs/current/functions-admin.html (pg_total_relation_size function)
# Don't include TOAST tables as their size is included in pg_total_relation_size for each table
# and is not interesting to us directly: https://www.postgresql.org/docs/current/storage-toast.html.
# Don't include indexes (relkind <> 'i') as their size is included in pg_total_relation_size as well.
"$BINDIR/psql" cfdb -c "SELECT nspname || '.' || relname AS relation,
    pg_size_pretty(pg_total_relation_size(C.oid)) AS total_size
  FROM pg_class C
  LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace)
  WHERE nspname NOT IN ('pg_catalog', 'information_schema')
  AND C.relkind <> 'i'
  AND nspname !~ '^pg_toast'
  ORDER BY pg_total_relation_size(C.oid) DESC
  Limit 20;" > "$tmpdir"/cfdb-biggest-tables.log
log_cmd "$BINDIR/psql cfdb -c \"SELECT count(*) as host_count from __hosts;\""
log_cmd "$BINDIR/psql cfdb -c \"SELECT count(*) as host_count_deleted from __hosts WHERE deleted IS NOT NULL;\""
log_cmd "$BINDIR/psql cfdb -c \"SELECT count(*) as host_count_reporting_last_two_weeks from __hosts WHERE lastreporttimestamp >= NOW() - INTERVAL '2 weeks';\""
log_cmd "$BINDIR/psql cfdb -c \"SELECT count(*) as host_count_reporting_last_month from __hosts WHERE lastreporttimestamp >= NOW() - INTERVAL '1 month';\""
log_cmd "$BINDIR/psql cfdb -c \"select name, details, units, avg(value), count(value) from diagnostics group by name, details, units;\""
log_cmd "$BINDIR/psql cfdb -c \"select host, status, type, count(*) from __status group by host, status, type order by host, status;\""
log_cmd "$BINDIR/psql cfdb -c \"select count(*) from pg_stat_activity;\""
log_cmd "$BINDIR/psql cfdb -c \"select max(now() - xact_start) from pg_stat_activity where state in ('idle in transaction', 'active');\""
"$BINDIR/psql" cfdb -c "SELECT now(),* from pg_stat_activity;" > "$tmpdir"/psql-current-queries.log
"$BINDIR/psql" cfdb -c "SELECT
    age(clock_timestamp(), query_start),
    usename,
    datname,
    query
FROM pg_stat_activity
WHERE
    state != 'idle'
AND query NOT ILIKE '%pg_stat_activity%'
ORDER BY age desc;" > "$tmpdir"/psql-age-of-nonidle-queries.log

# This can help you see queries that are not currently progressing because of other parts of the system (e.g., lock contention)
"$BINDIR/psql" cfdb -c "SELECT
    usename,
    datname,
    query,
    wait_event_type,
    wait_event
FROM pg_stat_activity
WHERE
    state != 'idle'
AND wait_event != ''" > "$tmpdir"/psql-waiting-not-idle.log

"$BINDIR/psql" cfdb -x -c "SELECT * FROM pg_stat_database" > "$tmpdir"/psql-db-statistics.log

# Table statistics; The numbers in these columns can help you evaluate how your
# indexes are performing and whether they're being effectively used by the
# queries you're running. If you find that your tables have many sequential
# scans, you would probably benefit from creating additional indexes that can be
# used by your most common queries.
"$BINDIR/psql" cfdb -x -c "SELECT * FROM pg_stat_all_tables" > "$tmpdir"/psql-table-statisctics.log

# This can be valuable information that can help you evaluate whether you would
# benefit from adding RAM to your database cluster so that your most common
# queries can be effectively cached.
"$BINDIR/psql" cfdb -c "SELECT
    datname,
    100 * blks_hit / (blks_hit + blks_read) as cache_hit_ratio
FROM
    pg_stat_database
WHERE
    (blks_hit + blks_read) > 0;" > "$tmpdir"/psql-database-cache-hit-ratio.log


# Get row count of each table https://stackoverflow.com/a/28709658
"$BINDIR/psql" cfdb -c "CREATE FUNCTION cf_support_rowcount_all(schema_name text default 'public')
  RETURNS table(table_name text, cnt bigint) as
\$\$
declare
 table_name text;
begin
  for table_name in SELECT c.relname FROM pg_class c
    JOIN pg_namespace s ON (c.relnamespace=s.oid)
    WHERE c.relkind = 'r' AND s.nspname=schema_name
  LOOP
    RETURN QUERY EXECUTE format('select cast(%L as text),count(*) from %I.%I',
       table_name, schema_name, table_name);
  END LOOP;
end
\$\$ language plpgsql;
select cf_support_rowcount_all();
DROP FUNCTION cf_support_rowcount_all;" > "$tmpdir"/psql-current-rows-per-table.log

# non_interactive defined in cf-support which sources this file so disable check
# shellcheck disable=SC2154
if [ "$non_interactive" -eq 0 ]; then
  read -r -p "Include masterfiles in support submission [Y/n]: " response
fi
response=${response:-y}
case $response in
  [yY][eE][sS]|[yY])
    mkdir -p "$tmpdir$WORKDIR"
    "$BINDIR/rsync" -az "$WORKDIR/masterfiles" "$tmpdir$WORKDIR"
    echo "Added directory $WORKDIR/masterfiles"
    ;;
  *)
    echo "Not including masterfiles in support submission"
    ;;
esac


# check on permissions of pid files
log_cmd "find /var/cfengine -name '*.pid' | xargs ls -l"
