Output error log
unknown
plain_text
3 years ago
18 kB
9
Indexable
# The status of the "scylla-server" service
[root@ip-10-3-73-209 ~]# systemctl status scylla-server.service
● scylla-server.service - Scylla Server
Loaded: loaded (/usr/lib/systemd/system/scylla-server.service; enabled; vendor preset: disabled)
Drop-In: /etc/systemd/system/scylla-server.service.d
└─capabilities.conf, mounts.conf
Active: failed (Result: exit-code) since Sun 2021-10-24 17:57:09 UTC; 6min ago
Process: 14498 ExecStopPost=/usr/lib/scylla/scylla_stop (code=exited, status=0/SUCCESS)
Process: 14496 ExecStart=/usr/bin/scylla $SCYLLA_ARGS $SEASTAR_IO $DEV_MODE $CPUSET (code=exited, status=1/FAILURE)
Process: 14468 ExecStartPre=/usr/lib/scylla/scylla_prepare (code=exited, status=0/SUCCESS)
Main PID: 14496 (code=exited, status=1/FAILURE)
Status: "creating commitlog directory"
Oct 24 17:57:09 ip-10-3-73-209.ec2.internal scylla[14496]: [shard 0] init - Scylla API server listening on 127.0.0.1:10000 ...
Oct 24 17:57:09 ip-10-3-73-209.ec2.internal scylla[14496]: [shard 0] init - I/O Scheduler is not properly configured! This is a non-supported setup, and per...bly bad.
Reason found: none of --max-io-requests, --io-properties and --io-properties-file are set.
To properly configure the I/O Scheduler, run the scylla_io_setup utility shipped with Scylla....
Oct 24 17:57:09 ip-10-3-73-209.ec2.internal scylla[14496]: [shard 0] init - Could not initialize /var/lib/scylla/commitlog: open: Permission denied
Oct 24 17:57:09 ip-10-3-73-209.ec2.internal scylla[14496]: [shard 0] init - Shutdown database started
Oct 24 17:57:09 ip-10-3-73-209.ec2.internal scylla[14496]: [shard 0] init - Shutdown database finished
Oct 24 17:57:09 ip-10-3-73-209.ec2.internal scylla[14496]: [shard 0] init - Startup failed: std::system_error (error system:13, open: Permission denied)
Oct 24 17:57:09 ip-10-3-73-209.ec2.internal systemd[1]: scylla-server.service: main process exited, code=exited, status=1/FAILURE
Oct 24 17:57:09 ip-10-3-73-209.ec2.internal systemd[1]: Failed to start Scylla Server.
Oct 24 17:57:09 ip-10-3-73-209.ec2.internal systemd[1]: Unit scylla-server.service entered failed state.
Oct 24 17:57:09 ip-10-3-73-209.ec2.internal systemd[1]: scylla-server.service failed.
Hint: Some lines were ellipsized, use -l to show in full.
scylla_io_setup
/usr/lib/scylla/libexec/scylla_io_setup:88: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.
cfg = yaml.load(open(os.path.join(conf_dir, "scylla.yaml")))
ERROR:root:Filesystem at /var/lib/scylla/data has only 2100834304 bytes available; that is less than the recommended 10 GB. Please free up space and run scylla_io_setup again.
_______________________________________________________________________________________________________________________________
# The status of the "scylla-jmx" service
[root@ip-10-3-73-209 ~]# systemctl status scylla-jmx
● scylla-jmx.service - Scylla JMX
Loaded: loaded (/usr/lib/systemd/system/scylla-jmx.service; enabled; vendor preset: disabled)
Active: failed (Result: exit-code) since Sun 2021-10-24 11:27:33 UTC; 1 day 3h ago
Main PID: 1958 (code=exited, status=143)
Oct 25 13:52:48 ip-10-3-73-209.ec2.internal systemd[1]: Dependency failed for Scylla JMX.
Oct 25 13:52:48 ip-10-3-73-209.ec2.internal systemd[1]: Job scylla-jmx.service/start failed with result 'dependency'.
Oct 25 13:58:02 ip-10-3-73-209.ec2.internal systemd[1]: Dependency failed for Scylla JMX.
Oct 25 13:58:02 ip-10-3-73-209.ec2.internal systemd[1]: Job scylla-jmx.service/start failed with result 'dependency'.
Oct 25 13:58:06 ip-10-3-73-209.ec2.internal systemd[1]: Dependency failed for Scylla JMX.
Oct 25 13:58:06 ip-10-3-73-209.ec2.internal systemd[1]: Job scylla-jmx.service/start failed with result 'dependency'.
Oct 25 14:59:38 ip-10-3-73-209.ec2.internal systemd[1]: Dependency failed for Scylla JMX.
Oct 25 14:59:38 ip-10-3-73-209.ec2.internal systemd[1]: Job scylla-jmx.service/start failed with result 'dependency'.
Oct 25 15:07:25 ip-10-3-73-209.ec2.internal systemd[1]: Dependency failed for Scylla JMX.
Oct 25 15:07:25 ip-10-3-73-209.ec2.internal systemd[1]: Job scylla-jmx.service/start failed with result 'dependency'.
_______________________________________________________________________________________________________________________________
# After running Scylla in the developer mode, I didn't receive any error related to the "I/O Scheduler" therefore the output of the "systemctl status scylla-server.service" looks different:
[root@ip-10-3-73-209 ~]# systemctl status scylla-server
● scylla-server.service - Scylla Server
Loaded: loaded (/usr/lib/systemd/system/scylla-server.service; enabled; vendor preset: disabled)
Drop-In: /etc/systemd/system/scylla-server.service.d
└─capabilities.conf, mounts.conf
Active: failed (Result: exit-code) since Sun 2021-10-24 21:18:30 UTC; 7min ago
Process: 20875 ExecStopPost=/usr/lib/scylla/scylla_stop (code=exited, status=0/SUCCESS)
Process: 20872 ExecStart=/usr/bin/scylla $SCYLLA_ARGS $SEASTAR_IO $DEV_MODE $CPUSET (code=exited, status=1/FAILURE)
Process: 25031 ExecStartPre=/usr/lib/scylla/scylla_prepare (code=exited, status=1/FAILURE)
Main PID: 20872 (code=exited, status=1/FAILURE)
Oct 24 21:18:30 ip-10-3-73-209.ec2.internal scylla_prepare[25031]: File "/opt/scylladb/python3/lib64/python3.7/configparser.py", line 717, in read_file
Oct 24 21:18:30 ip-10-3-73-209.ec2.internal scylla_prepare[25031]: self._read(f, source)
Oct 24 21:18:30 ip-10-3-73-209.ec2.internal scylla_prepare[25031]: File "/opt/scylladb/python3/lib64/python3.7/configparser.py", line 1110, in _read
Oct 24 21:18:30 ip-10-3-73-209.ec2.internal scylla_prepare[25031]: raise e
Oct 24 21:18:30 ip-10-3-73-209.ec2.internal scylla_prepare[25031]: configparser.ParsingError: Source contains parsing errors: '<???>'
Oct 24 21:18:30 ip-10-3-73-209.ec2.internal scylla_prepare[25031]: [line 48]: '--memory 2G \n'
Oct 24 21:18:30 ip-10-3-73-209.ec2.internal systemd[1]: scylla-server.service: control process exited, code=exited status=1
Oct 24 21:18:30 ip-10-3-73-209.ec2.internal systemd[1]: Failed to start Scylla Server.
Oct 24 21:18:30 ip-10-3-73-209.ec2.internal systemd[1]: Unit scylla-server.service entered failed state.
Oct 24 21:18:30 ip-10-3-73-209.ec2.internal systemd[1]: scylla-server.service failed.
[root@ip-10-3-73-209 ~]# systemctl restart scylla-server
Job for scylla-server.service failed because the control process exited with error code. See "systemctl status scylla-server.service" and "journalctl -xe" for details.
_______________________________________________________________________________________________________________________________
# Down below is the output of the "systemctl status" after I have added the "--memory" parameter in the /etc/sysconfig/scylla-server.
[root@ip-10-3-73-209 etc]# systemctl status scylla-server -l
● scylla-server.service - Scylla Server
Loaded: loaded (/usr/lib/systemd/system/scylla-server.service; enabled; vendor preset: disabled)
Drop-In: /etc/systemd/system/scylla-server.service.d
└─capabilities.conf, mounts.conf
Active: failed (Result: exit-code) since Mon 2021-10-25 07:27:21 UTC; 50s ago
Process: 24290 ExecStopPost=/usr/lib/scylla/scylla_stop (code=exited, status=0/SUCCESS)
Process: 24288 ExecStart=/usr/bin/scylla $SCYLLA_ARGS $SEASTAR_IO $DEV_MODE $CPUSET (code=exited, status=1/FAILURE)
Process: 24222 ExecStartPre=/usr/lib/scylla/scylla_prepare (code=exited, status=0/SUCCESS)
Main PID: 24288 (code=exited, status=1/FAILURE)
Oct 25 07:27:21 ip-10-3-73-209.ec2.internal scylla_prepare[24222]: Setting mask 00000001 in /sys/class/net/eth0/queues/tx-0/xps_cpus: failed to write into /sys/class/net/eth0/queues/tx-0/xps_cpus: (<class 'FileNotFoundError'>, FileNotFoundError(2, 'No such file or directory'), <traceback object at 0x7f02fcf6c108>)
Oct 25 07:27:21 ip-10-3-73-209.ec2.internal scylla_prepare[24222]: Writing '4096' to /proc/sys/net/core/somaxconn
Oct 25 07:27:21 ip-10-3-73-209.ec2.internal scylla_prepare[24222]: Writing '4096' to /proc/sys/net/ipv4/tcp_max_syn_backlog
Oct 25 07:27:21 ip-10-3-73-209.ec2.internal scylla_prepare[24222]: /usr/lib/scylla/scylla_util.py:419: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.
Oct 25 07:27:21 ip-10-3-73-209.ec2.internal scylla_prepare[24222]: y = yaml.load(open(scylla_yaml_name))
Oct 25 07:27:21 ip-10-3-73-209.ec2.internal scylla[24288]: Could not initialize seastar: std::runtime_error (insufficient physical memory: needed 2147483648 available 500000000)
Oct 25 07:27:21 ip-10-3-73-209.ec2.internal systemd[1]: scylla-server.service: main process exited, code=exited, status=1/FAILURE
Oct 25 07:27:21 ip-10-3-73-209.ec2.internal systemd[1]: Failed to start Scylla Server.
Oct 25 07:27:21 ip-10-3-73-209.ec2.internal systemd[1]: Unit scylla-server.service entered failed state.
Oct 25 07:27:21 ip-10-3-73-209.ec2.internal systemd[1]: scylla-server.service failed.
_______________________________________________________________________________________________________________________________
# The error log output of the "journalctl -xe"
[root@ip-10-3-73-209 ~]# journalctl -xe
-- Defined-By: systemd
-- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel
--
-- Unit scylla-server.service has failed.
--
-- The result is failed.
Oct 24 18:05:45 ip-10-3-73-209.ec2.internal systemd[1]: Dependency failed for Scylla JMX.
-- Subject: Unit scylla-jmx.service has failed
-- Defined-By: systemd
-- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel
--
-- Unit scylla-jmx.service has failed.
--
-- The result is dependency.
Oct 24 18:05:45 ip-10-3-73-209.ec2.internal systemd[1]: Job scylla-jmx.service/start failed with result 'dependency'.
Oct 24 18:05:45 ip-10-3-73-209.ec2.internal systemd[1]: Dependency failed for Run Scylla Housekeeping daily mode.
-- Subject: Unit scylla-housekeeping-daily.timer has failed
-- Defined-By: systemd
-- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel
--
-- Unit scylla-housekeeping-daily.timer has failed.
--
-- The result is dependency.
Oct 24 18:05:45 ip-10-3-73-209.ec2.internal systemd[1]: Job scylla-housekeeping-daily.timer/start failed with result 'dependency'.
Oct 24 18:05:45 ip-10-3-73-209.ec2.internal systemd[1]: Dependency failed for Run Scylla Housekeeping restart mode.
-- Subject: Unit scylla-housekeeping-restart.timer has failed
-- Defined-By: systemd
-- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel
--
-- Unit scylla-housekeeping-restart.timer has failed.
--
-- The result is dependency.
Oct 24 18:05:45 ip-10-3-73-209.ec2.internal systemd[1]: Job scylla-housekeeping-restart.timer/start failed with result 'dependency'.
Oct 24 18:05:45 ip-10-3-73-209.ec2.internal polkitd[730]: Unregistered Authentication Agent for unix-process:14616:2406020 (system bus name :1.75, object path /org/free
Oct 24 18:05:45 ip-10-3-73-209.ec2.internal systemd[1]: Unit scylla-server.service entered failed state.
Oct 24 18:05:45 ip-10-3-73-209.ec2.internal systemd[1]: scylla-server.service failed.
_______________________________________________________________________________________________________________________________
# The error log output of the "journalctl -u scylla-server"
[root@ip-10-3-73-209 ~]# journalctl -u scylla-server
-- Logs begin at Sun 2021-10-24 11:24:46 UTC, end at Sun 2021-10-24 18:18:35 UTC. --
Oct 24 11:26:04 ip-10-3-73-209.ec2.internal systemd[1]: Starting Scylla Server...
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: ERROR: Bad configuration mode (sq) and cpu-mask value (0x00000001): this results in a zero-mask for co
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: ERROR: Bad configuration mode (sq_split) and cpu-mask value (0x00000001): this results in a zero-mask
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: /usr/lib/scylla/libexec/perftune.py:1110: YAMLLoadWarning: calling yaml.load() without Loader=... is d
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: y = yaml.load(open(prog_args.options_file))
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: /usr/lib/scylla/libexec/perftune.py:900: DeprecationWarning: Will be removed in 1.0. Use equivalent De
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: udev_obj = pyudev.Device.from_device_number(self.__pyudev_ctx, 'block', os.stat(directory).st_dev)
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: /usr/lib/scylla/libexec/perftune.py:934: DeprecationWarning: Will be removed in 1.0. Use equivalent De
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: udev_obj = pyudev.Device.from_device_file(self.__pyudev_ctx, "/dev/{}".format(device))
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: /usr/lib/scylla/libexec/perftune.py:967: DeprecationWarning: Will be removed in 1.0. Use equivalent De
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: udev = pyudev.Device.from_device_file(pyudev.Context(), dev_node)
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: irqbalance is not running
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Setting non-NVMe disks: xvdb...
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Setting mask 00000001 in /proc/irq/55/smp_affinity
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Setting mask 00000001 in /proc/irq/56/smp_affinity
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Not setting I/O Scheduler for xvdb - required schedulers (['none', 'noop']) are not supported
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Writing '2' to /sys/devices/vbd-51728/block/xvdb/queue/nomerges
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: No NVMe disks to tune
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Setting a physical interface eth0...
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Distributing all IRQs
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Setting mask 00000001 in /proc/irq/57/smp_affinity
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Setting mask 00000001 in /sys/class/net/eth0/queues/rx-0/rps_cpus
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Setting net.core.rps_sock_flow_entries to 32768
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Setting limit 32768 in /sys/class/net/eth0/queues/rx-0/rps_flow_cnt
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Trying to enable ntuple filtering HW offload for eth0...not supported
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Setting mask 00000001 in /sys/class/net/eth0/queues/tx-0/xps_cpus: failed to write into /sys/class/net
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Writing '4096' to /proc/sys/net/core/somaxconn
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: Writing '4096' to /proc/sys/net/ipv4/tcp_max_syn_backlog
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: /usr/lib/scylla/scylla_util.py:419: YAMLLoadWarning: calling yaml.load() without Loader=... is depreca
Oct 24 11:26:05 ip-10-3-73-209.ec2.internal scylla_prepare[1548]: y = yaml.load(open(scylla_yaml_name))
Oct 24 11:26:11 ip-10-3-73-209.ec2.internal scylla[1646]: Scylla version 3.1.1-0.20191024.f32ec885c starting ...
Oct 24 11:26:11 ip-10-3-73-209.ec2.internal scylla[1646]: [shard 0] init - Scylla version 3.1.1-0.20191024.f32ec885c starting.
Oct 24 11:26:11 ip-10-3-73-209.ec2.internal scylla[1646]: [shard 0] init - Only 476 MiB per shard; this is below the recommended minimum of 1 GiB/shard; continuing sin
Oct 24 11:26:12 ip-10-3-73-209.ec2.internal scylla[1646]: [shard 0] snitch_logger - Ec2Snitch using region: us-east, zone: 1d.
Oct 24 11:26:12 ip-10-3-73-209.ec2.internal scylla[1646]: [shard 0] init - Scylla API server listening on 127.0.0.1:10000 ...
_______________________________________________________________________________________________________________________________
# The error log output of the "scylla_io_setup"
[root@ip-10-3-73-209 data]# scylla_io_setup
/usr/lib/scylla/libexec/scylla_io_setup:88: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.
cfg = yaml.load(open(os.path.join(conf_dir, "scylla.yaml")))
ERROR:root:Filesystem at /var/lib/scylla/data has only 2100834304 bytes available; that is less than the recommended 10 GB. Please free up space and run scylla_io_setup again.
# The above command was successfully executed after running Scylla in the developer mode.
_______________________________________________________________________________________________________________________________
# The error log output of the "scylla_setup"
[root@ip-10-3-73-209 data]# scylla_setup
Skip any of the following steps by answering 'no'
Do you want to run Amazon EC2 configuration check?
Yes - runs a script to verify that this instance is optimized for running Scylla. No - skips the configuration check.
[YES/no]
t2 doesn't support enhanced networking!
To enable enhanced networking, please use the instance type which supports it.
More documentation available at:
http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enhanced-networking.html#enabling_enhanced_networking
Traceback (most recent call last):
File "/usr/lib/scylla/libexec/scylla_setup", line 234, in <module>
run('/usr/lib/scylla/scylla_ec2_check --nic {}'.format(nic))
File "/usr/lib/scylla/scylla_util.py", line 286, in run
return subprocess.check_call(cmd, shell=shell, stdout=stdout, stderr=stderr, env=scylla_env)
File "/opt/scylladb/python3/lib64/python3.7/subprocess.py", line 347, in check_call
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['/usr/lib/scylla/scylla_ec2_check', '--nic', 'eth0']' returned non-zero exit status 1.Editor is loading...