/etc/perfsonar/psconfig/archives.d/http_logstash.json
{
"archiver": "http",
"data": {
"schema": 2,
"_url": "https://{% scheduled_by_address %}/logstash",
"op": "put",
"_headers": {
"x-ps-observer": "{% scheduled_by_address %}",
"content-type": "application/json", "Authorization":"Basic cGVy...."
}
}
}
/etc/httpd/conf.d/apache-logstash.conf
ProxyPass /logstash http://localhost:11283 status=+I
ProxyPassReverse /logstash http://localhost:11283 status=+I
ProxyPreserveHost On
<Location /logstash>
Authtype Basic
AuthUserFile /etc/perfsonar/opensearch/logstash_login
/etc/perfsonar/opensearch/logstash_login
perfsonar:$apr1....
# curl https://localhost/logstash \ -H "Authorization: Basic cGVy...." ok
/etc/logstash/pipelines.yml
- path.config: /etc/logstash/conf.d/*.conf pipeline.id: main - path.config: /usr/lib/perfsonar/logstash/pipeline/*.conf pipeline.id: pscheduler
# ls /usr/lib/perfsonar/logstash/pipeline 01-inputs.conf 04-iso8601-duration.conf 06-result_rtt.conf 99-outputs.conf 02-pscheduler_common.conf 05-geoip.conf 06-result_throughput.conf 03-normalize_endpoints.conf 06-result_latency.conf 06-result_trace.conf
/usr/lib/perfsonar/logstash/pipeline/01-inputs.conf
input {
http {
host => "localhost"
port => "11283" # ACII 112=p, 83=S
}
}
/usr/lib/perfsonar/logstash/pipeline/99-outputs.conf
output {
#If has [test][type] create index using that in name.
if [test][type] {
opensearch {
hosts => ["${opensearch_output_host}"]
ssl_certificate_verification => false
user => "${opensearch_output_user}"
password => "${opensearch_output_password}"
index => "pscheduler_%{[test][type]}-%{+YYYY.MM.dd}"
}
}
}
/etc/perfsonar/logstash/logstash_sysconfig
## Logstash environment variables. log_level=info opensearch_output_host=https://localhost:9200 opensearch_output_user=pscheduler_logstash opensearch_output_password=pscheduler_logstash XPACK_MONITORING_ENABLED=False LOGSTASH_ELASTIC_USER=pscheduler_logstash
/etc/perfsonar/opensearch/auth_setup.out
admin mE...... anomalyadmin vv...... kibanaserver qg...... kibanaro fc...... logstash Nm...... readall E4...... snapshotrestore MK...... pscheduler_logstash Ji...... pscheduler_reader YR...... pscheduler_writer IX......
# U=`cat /etc/perfsonar/opensearch/auth_setup.out | head -n 1 | sed -e 's/ /:/g'`
# alias CURL="curl -X GET -u $U"
# CURL 'https://127.0.0.1:9200/_cluster/health?pretty'
{
"cluster_name" : "opensearch",
"status" : "yellow",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"discovered_master" : true,
"discovered_cluster_manager" : true,
"active_primary_shards" : 501,
"active_shards" : 501,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 499,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 50.1
}
# CURL 'https://127.0.0.1:9200/_cat/indices?v' health status index uuid pri rep docs.count docs.deleted store.size pri.store.size yellow open pscheduler_latencybg-2023.05.10 LlDBhY_ES1-nuONvTHdhSg 1 1 29536 0 38.5mb 38.5mb ... yellow open pscheduler_latencybg-2023.09.04 eKlsY1_cRTqfnBxgkekChg 1 1 20246 0 24.4mb 24.4mb yellow open pscheduler_throughput-2023.05.10 pKT63gJAR2aoMDaLREm3GQ 1 1 94 0 880.3kb 880.3kb ... yellow open pscheduler_throughput-2023.09.03 vijnOOjITk67a7mAZzX2IA 1 1 42 0 391.1kb 391.1kb yellow open pscheduler_trace-2023.05.11 NwXrbs5fRBqhUbIgZsgvNw 1 1 1589 0 2.4mb 2.4mb ... yellow open pscheduler_trace-2023.09.03 VnBhB_bkTjmpL_oYuXhyBw 1 1 4171 0 6.5mb 6.5mb
# CURL 'https://localhost:9200/pscheduler_throughput-2023.09.03/_search' -H "Content-Type: application/json" -d '{ "size": 3 }'
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 139,
"relation": "eq"
},
"max_score": 1,
"hits": [
{ /*...*/ },
{ /*...*/ },
{ /*...*/ }
]
}
}
/etc/httpd/conf.d/apache-elmond.conf
SSLProxyEngine On
ProxyPass /esmond/perfsonar/archive http://localhost:5000 status=+I
ProxyPreserveHost On
Header always set Access-Control-Allow-Origin "*"
# netstat -anp | grep 5000 | grep LISTEN tcp 0 0 127.0.0.1:5000 0.0.0.0:* LISTEN 2159/python3 # cat /proc/2159/cmdline /usr/bin/python3 /usr/lib/perfsonar/elmond/app.py
https://github.com/perfsonar/elmond
elmond/elmond/perfsonar-elmond/elmond/app.py
elmond/elmond/perfsonar-elmond/elmond/data.py
⭐ Only opensearch used
https://ps-daej.kreonet2.net/perfsonar-graphs/ ?source=134.75.207.2 &dest=138.44.228.7 &url=https://ps-daej.kreonet2.net/esmond/perfsonar/archive/ &timeframe=1d#start=1693273543&end=1693878343 A=https://ps-daej.kreonet2.net/esmond/perfsonar/archive/ B=b01012273dd3c9c66e8bbfc6b095dde515c57cbb11afa76a4015dca4bfee7da2 C=e632505ff9396efd34f63927cace884b5cb14cc4b5bf61d1b431c182a56fca0e curl 'A/?source=134.75.207.2&destination=138.44.228.7' curl 'A/B/failures/base/0 ?time-start=1693273543&time-end=1693878343' curl 'A/B/packet-count-sent/aggregations/86400 ?time-start=1693273543&time-end=1693878343' curl 'A/B/histogram-owdelay/statistics/86400 ?time-start=1693273543&time-end=1693878343' curl 'A/B/packet-loss-rate/aggregations/86400 ?time-start=1693273543&time-end=1693878343' curl 'A/B/packet-count-lost/aggregations/86400 ?time-start=1693273543&time-end=1693878343' curl 'A/C/throughput/base/0 ?time-start=1693273543&time-end=1693878343'
# du -h /var/lib/pgsql 2.9G /var/lib/pgsql # du -h /var/lib/opensearch 15G /var/lib/opensearch
# su postgres
bash-4.2$ psql
psql (10.17)
postgres=# \l
List of databases
Name | Owner | Encoding | Collate | Ctype | Access privileges
------------+------------+-----------+---------+-------+---------------------------
esmond | postgres | SQL_ASCII | C | C | =Tc/postgres +
| | | | | postgres=CTc/postgres +
| | | | | esmond=CTc/postgres
pscheduler | pscheduler | SQL_ASCII | C | C | =Tc/pscheduler +
| | | | | pscheduler=CTc/pscheduler
postgres=# \c esmond
You are now connected to database "esmond" as user "postgres".
esmond=# select
esmond-# table_name,
esmond-# pg_size_pretty(pg_relation_size(quote_ident(table_name))),
esmond-# pg_relation_size(quote_ident(table_name))
esmond-# from information_schema.tables
esmond-# where table_schema = 'public';
table_name | pg_size_pretty | pg_relation_size
----------------------------+----------------+------------------
django_migrations | 8192 bytes | 8192
ps_metadata_parameters | 8192 bytes | 8192
ps_networkelement_subject | 0 bytes | 0
django_content_type | 8192 bytes | 8192
auth_group_permissions | 0 bytes | 0
auth_group | 0 bytes | 0
auth_user | 8192 bytes | 8192
auth_user_groups | 0 bytes | 0
auth_permission | 8192 bytes | 8192
auth_user_user_permissions | 0 bytes | 0
django_admin_log | 0 bytes | 0
ps_p2p_subject | 8192 bytes | 8192
useripaddress | 0 bytes | 0
ps_metadata | 8192 bytes | 8192
ps_event_types | 32 kB | 32768
authtoken_token | 8192 bytes | 8192
django_session | 0 bytes | 0
(17 rows)
esmond=# \c pscheduler
You are now connected to database "pscheduler" as user "postgres".
pscheduler=# select
pscheduler-# table_name,
pscheduler-# pg_size_pretty(pg_relation_size(quote_ident(table_name))),
pscheduler-# pg_relation_size(quote_ident(table_name))
pscheduler-# from information_schema.tables
pscheduler-# where table_schema = 'public';
table_name | pg_size_pretty | pg_relation_size
---------------------------+----------------+------------------
scheduling_class | 8192 bytes | 8192
archive_default | 16 kB | 16384
tool_test | 8192 bytes | 8192
table_version | 8192 bytes | 8192
heartbeat | 528 kB | 540672
auth_key_type | 8192 bytes | 8192
auth_key | 0 bytes | 0
control | 8192 bytes | 8192
queries | 0 bytes | 0
heartbeat_full | 0 bytes | 0
archiving_eligible | 0 bytes | 0
schedule | 0 bytes | 0
archiver_test | 0 bytes | 0
heartbeat_json | 0 bytes | 0
http_queue | 48 kB | 49152
test | 440 kB | 450560
tool | 696 kB | 712704
archiver_summary | 0 bytes | 0
context | 40 kB | 40960
run_conflictable | 0 bytes | 0
run_status | 0 bytes | 0
run_status_short | 0 bytes | 0
schedule_upcoming | 0 bytes | 0
schedule_runs_to_schedule | 0 bytes | 0
schedule_overlap | 0 bytes | 0
run_state | 8192 bytes | 8192
task | 7552 kB | 7733248
configurables | 8192 bytes | 8192
run_latest | 136 kB | 139264
archiver | 216 kB | 221184
run | 388 MB | 406921216
archiving | 575 MB | 602931200
(32 rows)
https://stackoverflow.com/questions/21738408/postgresql-list-and-order-tables-by-size