EMR Jupyterhub の Notebookファイルはどこに保管されているのか
備忘録的な
https://docs.aws.amazon.com/ja_jp/emr/latest/ReleaseGuide/emr-jupyterhub-considerations.html に書いてあるとおりJupyterHubでは 永続化設定をしてない場合自分でcronで保存してね。との事。
どこにあるかは以下の手順で確認できる。
ssh hadoop@emr-instance
sudo docker inspect jupyterhub
[
{
"Id": "b12c789d1e2044c88cbe33e4ce1c6521b0662611cb6eec80e75f9c5b4c046104",
"Created": "2020-02-29T06:38:28.166983484Z",
"Path": "tini",
"Args": [
"-g",
"--",
"jupyterhub"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 14294,
"ExitCode": 0,
"Error": "",
"StartedAt": "2020-02-29T06:38:28.76309949Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:e28d92801f6ee34cfdd847563529d32c8d17f507229e55fff446a8257fa586a5",
"ResolvConfPath": "/mnt/var/lib/docker/containers/b12c789d1e2044c88cbe33e4ce1c6521b0662611cb6eec80e75f9c5b4c046104/resolv.conf",
"HostnamePath": "/mnt/var/lib/docker/containers/b12c789d1e2044c88cbe33e4ce1c6521b0662611cb6eec80e75f9c5b4c046104/hostname",
"HostsPath": "/mnt/var/lib/docker/containers/b12c789d1e2044c88cbe33e4ce1c6521b0662611cb6eec80e75f9c5b4c046104/hosts",
"LogPath": "/mnt/var/lib/docker/containers/b12c789d1e2044c88cbe33e4ce1c6521b0662611cb6eec80e75f9c5b4c046104/b12c789d1e2044c88cbe33e4ce1c6521b0662611cb6eec80e75f9c5b4c046104-json.log",
"Name": "/jupyterhub",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/var/lib/jupyter/home:/home",
"/var/log/jupyter:/var/log/jupyter",
"/etc/jupyter:/etc/jupyter"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "default",
"PortBindings": {
"9443/tcp": [
{
"HostIp": "",
"HostPort": "9443"
}
]
},
"RestartPolicy": {
"Name": "on-failure",
"MaximumRetryCount": 5
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "shareable",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": true,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": [
"label=disable"
],
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": [],
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": 0,
"Ulimits": [
{
"Name": "nofile",
"Hard": 4096,
"Soft": 1024
}
],
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": null,
"ReadonlyPaths": null
},
"GraphDriver": {
"Data": {
"LowerDir": "/mnt/var/lib/docker/overlay2/3c409c255334c665d2fc2e5359d4ab2dc53b58fc1dc56e5498afa68f42de8d32-init/diff:/mnt/var/lib/docker/overlay2/39f4bd5781917f9d41114bf008920ff88c846cc0b06d38005bc063973290c7e0/diff:/mnt/var/lib/docker/overlay2/857c34bf8080d83361fab861945ef96ef3d58e40085bb21d9f0d9325a48a6305/diff:/mnt/var/lib/docker/overlay2/fe9149e14de47b33a211e1b32df6b02f220080625acb425ea7f330600379c62b/diff:/mnt/var/lib/docker/overlay2/bf9c0d7847f6987e689efbd802c5fb96db72b6175c45d39c99827da566e55766/diff:/mnt/var/lib/docker/overlay2/867648c5f6c13cd74d76407d92d14a2d5ca6e2d31712d8340cb9ccea06d3816c/diff:/mnt/var/lib/docker/overlay2/5806e4f02408ee9510ca8629fb47a8fa970b03d54c1ea1e40206f587fa5d90a1/diff:/mnt/var/lib/docker/overlay2/d96f9f7ee61d8632a67fb5740a76c6598c74b2a691179f4879e8e52359864452/diff:/mnt/var/lib/docker/overlay2/cbf452e49c19178f236db5d9a1921ce72c28d1cd11b2719598acb68c0e0495ae/diff:/mnt/var/lib/docker/overlay2/4a60b046e9c871dc35dc6cd141f8d632e59ae0461075ac5861f57719f3b9c014/diff:/mnt/var/lib/docker/overlay2/05859c9d7b3bf11b5fdb19e42ab7846f3c182412f3610d746e1568df9ba6d883/diff:/mnt/var/lib/docker/overlay2/c16bfb4bb3c63de845c05969a715d90582dce39b2ee3955f37ae4d47cd9f3cb9/diff:/mnt/var/lib/docker/overlay2/183edb636d260620640b13a8a1dce857ee1190f5484f453083ba723924da1977/diff:/mnt/var/lib/docker/overlay2/51e07d9f302c872dc477df04a31d09d0b033c6b138ed33537b96583f39b837c7/diff:/mnt/var/lib/docker/overlay2/9929f425eea29ae3c1c197bbae93896324059d582e95ce03f60a21ef88b9b2c7/diff:/mnt/var/lib/docker/overlay2/e796d35fdeb59ec98eeac8593b9a722bfaa271df07c0af02e1b0b79f519f5f22/diff:/mnt/var/lib/docker/overlay2/733f357bc2e355c2cbf8a1500bf590527764cd7372cda87376d9cf92df7b0a33/diff:/mnt/var/lib/docker/overlay2/02431a6e441851d9fbc6b677d27c4920e8231d9c1cdc9dd19b60989262b361d0/diff:/mnt/var/lib/docker/overlay2/deae1599d76644be7ac6f783fdc5d17970188a914e17287451eeee6065915dfb/diff:/mnt/var/lib/docker/overlay2/fb17b9c311885dd69e3e622053f82d9edec84a7d8b5bb77dfaa7d6e7f19cc550/diff:/mnt/var/lib/docker/overlay2/0043c53aeb70f0eb72506c4e67a1e73b4febeaca6e6389f5dc150d52667a4930/diff:/mnt/var/lib/docker/overlay2/1f812f8dd3a1efc9800be3f0f019604fbe8eb70e1a5edee3f5699779c01bed07/diff:/mnt/var/lib/docker/overlay2/9c87c6693667a6eb38042703d13e7ca3d286bc892c4f389d1597132c5356c1d5/diff:/mnt/var/lib/docker/overlay2/c33b8002784d8ef74734fd613515eaa82e1f0e841300a7a2d58a51781126b7ec/diff:/mnt/var/lib/docker/overlay2/24a64683b2a24a46f0bab5f4a28bb383f4319b044a53989eb7652d38fc103a9b/diff:/mnt/var/lib/docker/overlay2/04cfa3a48a64d796a596ffbf1bfbd13c0cbab0583cc1215e51e9d2162a84300b/diff:/mnt/var/lib/docker/overlay2/4d620421254a6508dd468f2edcc79e78d5dd20c7c25b5d921f96358fa38f0330/diff:/mnt/var/lib/docker/overlay2/5aa371fde9e9cee618ebec6586f44700cddedfda6aae989dbfb306c3b9f714a1/diff:/mnt/var/lib/docker/overlay2/e0ce97f1fba551ea8aeb2834b88334f6ffffee738bd6a2db05a44e5b1b155081/diff:/mnt/var/lib/docker/overlay2/c61fa0f9ed68cf8168b74ff8d3aa27465427c8f783988423b17339503606b44a/diff:/mnt/var/lib/docker/overlay2/62dfd7f2af71bce01b62ffaebe1375a8843610b0425e15de6fcb073a4267786e/diff:/mnt/var/lib/docker/overlay2/67ccfe8ee3a6ef3392ed168b944cb73c2359c6ed68c0ed0004f9134acce5392d/diff:/mnt/var/lib/docker/overlay2/2692821e480f801b2936b8f6bf7c605ca53a43b3b710520314616abf0febccb4/diff:/mnt/var/lib/docker/overlay2/135d286d9ce8149513fa0858b9d243cc569adf4316d05e35a48a4abe2b49914a/diff:/mnt/var/lib/docker/overlay2/77f6d080195f750b7c570103d00e6edca01aa0e129d34182485a6f3bc0acda15/diff",
"MergedDir": "/mnt/var/lib/docker/overlay2/3c409c255334c665d2fc2e5359d4ab2dc53b58fc1dc56e5498afa68f42de8d32/merged",
"UpperDir": "/mnt/var/lib/docker/overlay2/3c409c255334c665d2fc2e5359d4ab2dc53b58fc1dc56e5498afa68f42de8d32/diff",
"WorkDir": "/mnt/var/lib/docker/overlay2/3c409c255334c665d2fc2e5359d4ab2dc53b58fc1dc56e5498afa68f42de8d32/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "bind",
"Source": "/etc/jupyter",
"Destination": "/etc/jupyter",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/var/lib/jupyter/home",
"Destination": "/home",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/var/log/jupyter",
"Destination": "/var/log/jupyter",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
}
],
"Config": {
"Hostname": "jupyterhub",
"Domainname": "",
"User": "root",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"8888/tcp": {},
"9443/tcp": {}
},
"Tty": true,
"OpenStdin": true,
"StdinOnce": false,
"Env": [
"GRANT_SUDO=yes",
"PATH=/usr/local/bin:/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"DEBIAN_FRONTEND=noninteractive",
"CONDA_DIR=/opt/conda",
"SHELL=/bin/bash",
"NB_USER=jovyan",
"NB_UID=1000",
"NB_GID=100",
"LC_ALL=en_US.UTF-8",
"LANG=en_US.UTF-8",
"LANGUAGE=en_US.UTF-8",
"HOME=/home/jovyan",
"MINICONDA_VERSION=4.5.4",
"SPARKMAGIC_VERSION=0.12.*",
"SPARKMAGIC_KERNEL_DIRECTORY=/opt/conda/lib/python3.6/site-packages/sparkmagic/kernels",
"S3CONTENTS_VERSION=0.1.13",
"JUPYTERLAB_VERSION=1.0*",
"JUPYTERHUB_VERSION=1.0.0",
"NOTEBOOK_VERSION=5.7.8"
],
"Cmd": [
"jupyterhub"
],
"ArgsEscaped": true,
"Image": "emr/jupyter-notebook:5.7.8",
"Volumes": null,
"WorkingDir": "/home/jovyan",
"Entrypoint": [
"tini",
"-g",
"--"
],
"OnBuild": null,
"Labels": {
"maintainer": "Jupyter Project <jupyter@googlegroups.com>"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "ac24e8c71881d50abd1103d2192db28be8b0732f7e0f1ba6447a36157cb4f549",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {
"8888/tcp": null,
"9443/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "9443"
}
]
},
"SandboxKey": "/var/run/docker/netns/ac24e8c71881",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "0800d2ca31c81d0f8a9966f48f09200cdb4a26d6fcf55dc588927a4d38121aca",
"Gateway": "172.17.0.1",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "172.17.0.2",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"MacAddress": "02:42:ac:11:00:02",
"Networks": {
"bridge": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"NetworkID": "1a2bef57a5199b3ff6964165525126aacb884ba633acf66d47278638986762bc",
"EndpointID": "0800d2ca31c81d0f8a9966f48f09200cdb4a26d6fcf55dc588927a4d38121aca",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.2",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "02:42:ac:11:00:02",
"DriverOpts": null
},
"emr-docker-bridge": {
"IPAMConfig": {},
"Links": null,
"Aliases": [
"b12c789d1e20"
],
"NetworkID": "93e00b6aee32f3204e8f99ce7b342f0fabc4784ea7d643a3f0be66ad68a1935c",
"EndpointID": "af289e1dda568046993107aadddbcfc19a44d87f50d7b48b011e8b87822db563",
"Gateway": "172.18.0.1",
"IPAddress": "172.18.0.2",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "02:42:ac:12:00:02",
"DriverOpts": null
}
}
}
}
]
"HostConfig": {
"Binds": [
"/var/lib/jupyter/home:/home",
"/var/log/jupyter:/var/log/jupyter",
"/etc/jupyter:/etc/jupyter"
],
/var/lib/jupyter/home に保存されていました。
