2024-01-20T21:30:34.555154172Z level=info ts=2024-01-20T21:30:34.554Z caller=main.go:389 msg="No time or size retention was set so using the default time retention" duration=15d 2024-01-20T21:30:34.555189917Z level=info ts=2024-01-20T21:30:34.554Z caller=main.go:443 msg="Starting Prometheus" version="(version=2.28.1, branch=HEAD, revision=b0944590a1c9a6b35dc5a696869f75f422b107a1)" 2024-01-20T21:30:34.555194332Z level=info ts=2024-01-20T21:30:34.554Z caller=main.go:448 build_context="(go=go1.16.5, user=root@2915dd495090, date=20210701-15:20:10)" 2024-01-20T21:30:34.555197458Z level=info ts=2024-01-20T21:30:34.554Z caller=main.go:449 host_details="(Linux 5.4.0-107-generic #121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022 x86_64 prometheus-0 (none))" 2024-01-20T21:30:34.555200469Z level=info ts=2024-01-20T21:30:34.554Z caller=main.go:450 fd_limits="(soft=1048576, hard=1048576)" 2024-01-20T21:30:34.555203276Z level=info ts=2024-01-20T21:30:34.554Z caller=main.go:451 vm_limits="(soft=unlimited, hard=unlimited)" 2024-01-20T21:30:34.569959298Z level=info ts=2024-01-20T21:30:34.556Z caller=web.go:541 component=web msg="Start listening for connections" address=0.0.0.0:9090 2024-01-20T21:30:34.569980933Z level=info ts=2024-01-20T21:30:34.557Z caller=main.go:824 msg="Starting TSDB ..." 2024-01-20T21:30:34.569984442Z level=info ts=2024-01-20T21:30:34.558Z caller=tls_config.go:191 component=web msg="TLS is disabled." http2=false 2024-01-20T21:30:34.569987884Z level=info ts=2024-01-20T21:30:34.562Z caller=head.go:780 component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-01-20T21:30:34.569991029Z level=info ts=2024-01-20T21:30:34.562Z caller=head.go:794 component=tsdb msg="On-disk memory mappable chunks replay completed" duration=5.07µs 2024-01-20T21:30:34.569994825Z level=info ts=2024-01-20T21:30:34.562Z caller=head.go:800 component=tsdb msg="Replaying WAL, this may take a while" 2024-01-20T21:30:34.569997709Z level=info ts=2024-01-20T21:30:34.562Z caller=head.go:854 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-01-20T21:30:34.570000691Z level=info ts=2024-01-20T21:30:34.562Z caller=head.go:860 component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.2µs wal_replay_duration=223.132µs total_replay_duration=260.945µs 2024-01-20T21:30:34.570003848Z level=info ts=2024-01-20T21:30:34.564Z caller=main.go:851 fs_type=EXT4_SUPER_MAGIC 2024-01-20T21:30:34.570006715Z level=info ts=2024-01-20T21:30:34.564Z caller=main.go:854 msg="TSDB started" 2024-01-20T21:30:34.570009563Z level=info ts=2024-01-20T21:30:34.564Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-20T21:30:34.570012737Z level=info ts=2024-01-20T21:30:34.565Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.19502ms remote_storage=26.813µs web_handler=330ns query_engine=4.36µs scrape=445.948µs scrape_sd=106.723µs notify=46.222µs notify_sd=39.43µs rules=4.916µs 2024-01-20T21:30:34.570016321Z level=info ts=2024-01-20T21:30:34.565Z caller=main.go:796 msg="Server is ready to receive web requests." 2024-01-20T21:35:48.493968688Z level=info ts=2024-01-20T21:35:48.493Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-20T21:35:48.495443774Z level=info ts=2024-01-20T21:35:48.495Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.605155ms remote_storage=6.461µs web_handler=4.157µs query_engine=4.451µs scrape=187.607µs scrape_sd=52.294µs notify=32.392µs notify_sd=10.886µs rules=7.286µs 2024-01-20T21:35:58.540311809Z level=info ts=2024-01-20T21:35:58.540Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-20T21:35:58.540904756Z level=info ts=2024-01-20T21:35:58.540Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=647.847µs remote_storage=3.171µs web_handler=882ns query_engine=1.874µs scrape=110.497µs scrape_sd=38.364µs notify=19.683µs notify_sd=23.122µs rules=2.381µs 2024-01-21T00:30:54.165429909Z level=info ts=2024-01-21T00:30:54.164Z caller=compact.go:518 component=tsdb msg="write block" mint=1705786254125 maxt=1705788000000 ulid=01HMMPTVNP3H1WZ1H8HF5CXN69 duration=30.887107ms 2024-01-21T00:30:54.167743174Z level=info ts=2024-01-21T00:30:54.167Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=1.890133ms 2024-01-21T01:00:09.182248367Z level=info ts=2024-01-21T01:00:09.181Z caller=compact.go:518 component=tsdb msg="write block" mint=1705788009125 maxt=1705795200000 ulid=01HMMRGDHJBEXXACPWZB1HWX7R duration=43.650202ms 2024-01-21T01:00:09.187564343Z level=info ts=2024-01-21T01:00:09.187Z caller=head.go:967 component=tsdb msg="Head GC completed" duration=4.458114ms 2024-01-21T01:07:54.929502270Z level=info ts=2024-01-21T01:07:54.929Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-21T01:07:54.930905152Z level=info ts=2024-01-21T01:07:54.930Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.524408ms remote_storage=2.9µs web_handler=25.114µs query_engine=1.917µs scrape=114.248µs scrape_sd=71.184µs notify=105.129µs notify_sd=30.31µs rules=7.92µs 2024-01-21T01:09:19.529137060Z level=info ts=2024-01-21T01:09:19.528Z caller=main.go:981 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-01-21T01:09:19.530454526Z level=info ts=2024-01-21T01:09:19.530Z caller=main.go:1012 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=1.433616ms remote_storage=6.288µs web_handler=2.957µs query_engine=1.349µs scrape=310.227µs scrape_sd=49.846µs notify=278.293µs notify_sd=44.467µs rules=9.106µs 2024-01-20T21:34:46.192172854Z Created MongoClient to connect to MongoDB! 2024-01-20T21:34:46.192265289Z Refreshing prometheus config file for first time 2024-01-20T21:35:16.221006717Z Error in configuration attempt! Number of tries: 1/3 2024-01-20T21:35:16.221072582Z No replica set members match selector "", Timeout: 30s, Topology Description: ]> 2024-01-20T21:35:21.226316075Z Refreshing prometheus config file for first time 2024-01-20T21:35:48.482094129Z Jobs detected : 0 2024-01-20T21:35:48.486972583Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-20T21:35:48.488405627Z alerting: 2024-01-20T21:35:48.488417575Z alertmanagers: 2024-01-20T21:35:48.488421212Z - static_configs: 2024-01-20T21:35:48.488424520Z - targets: null 2024-01-20T21:35:48.488427685Z global: 2024-01-20T21:35:48.488430844Z evaluation_interval: 15s 2024-01-20T21:35:48.488434263Z scrape_interval: 15s 2024-01-20T21:35:48.488437281Z rule_files: null 2024-01-20T21:35:48.488440309Z scrape_configs: 2024-01-20T21:35:48.488443357Z - job_name: mon_exporter 2024-01-20T21:35:48.488446371Z static_configs: 2024-01-20T21:35:48.488449376Z - targets: 2024-01-20T21:35:48.488452443Z - mon:8000 2024-01-20T21:35:48.488455448Z 2024-01-20T21:35:48.489770677Z New config written, updating prometheus 2024-01-20T21:35:53.523383673Z Prometheus config update successful 2024-01-20T21:35:58.526040013Z Refreshing prometheus config file for first time 2024-01-20T21:35:58.527971779Z Jobs detected : 1 2024-01-20T21:35:58.535330239Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-20T21:35:58.536650382Z alerting: 2024-01-20T21:35:58.536674473Z alertmanagers: 2024-01-20T21:35:58.536681677Z - static_configs: 2024-01-20T21:35:58.536687770Z - targets: null 2024-01-20T21:35:58.536693708Z global: 2024-01-20T21:35:58.536699864Z evaluation_interval: 15s 2024-01-20T21:35:58.536705811Z scrape_interval: 15s 2024-01-20T21:35:58.536711570Z rule_files: null 2024-01-20T21:35:58.536717339Z scrape_configs: 2024-01-20T21:35:58.536723069Z - job_name: mon_exporter 2024-01-20T21:35:58.536728863Z static_configs: 2024-01-20T21:35:58.536734714Z - targets: 2024-01-20T21:35:58.536740599Z - mon:8000 2024-01-20T21:35:58.536746539Z 2024-01-20T21:35:58.537648834Z New config written, updating prometheus 2024-01-20T21:36:03.577708208Z Prometheus config update successful 2024-01-20T21:36:08.587490172Z Listening to changes in prometheus jobs collection 2024-01-21T01:07:54.901292036Z Change detected, updating prometheus config 2024-01-21T01:07:54.901504184Z {'_id': {'_data': '8265AC6E6A000000042B022C0100296E5A1004A3876AC0B2964D7D8769E4A769CD5E3246645F6964006465AC6E6A101F7102DCDD78220004'}, 'operationType': 'insert', 'clusterTime': Timestamp(1705799274, 4), 'fullDocument': {'_id': ObjectId('65ac6e6a101f7102dcdd7822'), 'job_name': '3f97a5f674db463fb2809f2c4ee2b97f', 'metrics_path': '/snmp', 'nsr_id': '9627c7ac-330c-405b-a19d-e99e9b2d6de0', 'params': {'module': ['osm-snmp']}, 'relabel_configs': [{'source_labels': ['__address__'], 'target_label': '__param_target'}, {'source_labels': ['__param_target'], 'target_label': 'instance'}, {'target_label': '__address__', 'replacement': 'eechart-0000984742-snmpexporter:80'}], 'static_configs': [{'targets': ['172.21.249.255']}], 'vnfr_id': '3f97a5f674db463fb2809f2c4ee2b97f'}, 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('65ac6e6a101f7102dcdd7822')}} 2024-01-21T01:07:54.904824405Z Jobs detected : 2 2024-01-21T01:07:54.914705489Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-21T01:07:54.919453326Z alerting: 2024-01-21T01:07:54.919490198Z alertmanagers: 2024-01-21T01:07:54.919500714Z - static_configs: 2024-01-21T01:07:54.919509716Z - targets: null 2024-01-21T01:07:54.919518578Z global: 2024-01-21T01:07:54.919527305Z evaluation_interval: 15s 2024-01-21T01:07:54.919536124Z scrape_interval: 15s 2024-01-21T01:07:54.919544805Z rule_files: null 2024-01-21T01:07:54.919553928Z scrape_configs: 2024-01-21T01:07:54.919562646Z - job_name: mon_exporter 2024-01-21T01:07:54.919571360Z static_configs: 2024-01-21T01:07:54.919580332Z - targets: 2024-01-21T01:07:54.919589206Z - mon:8000 2024-01-21T01:07:54.919597988Z - job_name: 3f97a5f674db463fb2809f2c4ee2b97f 2024-01-21T01:07:54.919606941Z metrics_path: /snmp 2024-01-21T01:07:54.919615612Z params: 2024-01-21T01:07:54.919624361Z module: 2024-01-21T01:07:54.919633025Z - osm-snmp 2024-01-21T01:07:54.919641662Z relabel_configs: 2024-01-21T01:07:54.919650364Z - source_labels: 2024-01-21T01:07:54.919659064Z - __address__ 2024-01-21T01:07:54.919667747Z target_label: __param_target 2024-01-21T01:07:54.919676511Z - source_labels: 2024-01-21T01:07:54.919685178Z - __param_target 2024-01-21T01:07:54.919693955Z target_label: instance 2024-01-21T01:07:54.919702694Z - replacement: eechart-0000984742-snmpexporter:80 2024-01-21T01:07:54.919711576Z target_label: __address__ 2024-01-21T01:07:54.919720263Z static_configs: 2024-01-21T01:07:54.919755019Z - targets: 2024-01-21T01:07:54.919763172Z - 172.21.249.255 2024-01-21T01:07:54.919771017Z 2024-01-21T01:07:54.923308551Z New config written, updating prometheus 2024-01-21T01:07:59.961353051Z Prometheus config update successful 2024-01-21T01:07:59.965349085Z 2024-01-21T01:09:19.515395185Z Change detected, updating prometheus config 2024-01-21T01:09:19.515427373Z {'_id': {'_data': '8265AC6EBF000000092B022C0100296E5A1004A3876AC0B2964D7D8769E4A769CD5E3246645F6964006465AC6E6A101F7102DCDD78220004'}, 'operationType': 'delete', 'clusterTime': Timestamp(1705799359, 9), 'ns': {'db': 'osm', 'coll': 'prometheus_jobs'}, 'documentKey': {'_id': ObjectId('65ac6e6a101f7102dcdd7822')}} 2024-01-21T01:09:19.517429861Z Jobs detected : 1 2024-01-21T01:09:19.522203938Z Writing new config file to /etc/prometheus/prometheus.yml 2024-01-21T01:09:19.523765377Z alerting: 2024-01-21T01:09:19.523779464Z alertmanagers: 2024-01-21T01:09:19.523783355Z - static_configs: 2024-01-21T01:09:19.523786932Z - targets: null 2024-01-21T01:09:19.523799825Z global: 2024-01-21T01:09:19.523803451Z evaluation_interval: 15s 2024-01-21T01:09:19.523806757Z scrape_interval: 15s 2024-01-21T01:09:19.523810037Z rule_files: null 2024-01-21T01:09:19.523813371Z scrape_configs: 2024-01-21T01:09:19.523816576Z - job_name: mon_exporter 2024-01-21T01:09:19.523819827Z static_configs: 2024-01-21T01:09:19.523823089Z - targets: 2024-01-21T01:09:19.523826393Z - mon:8000 2024-01-21T01:09:19.523829615Z 2024-01-21T01:09:19.525145091Z New config written, updating prometheus 2024-01-21T01:09:24.559800974Z Prometheus config update successful 2024-01-21T01:09:24.561838916Z