Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

wasm operator always restart #243

Open
xiaowujee opened this issue Mar 11, 2021 · 1 comment
Open

wasm operator always restart #243

xiaowujee opened this issue Mar 11, 2021 · 1 comment
Labels
question Further information is requested

Comments

@xiaowujee
Copy link

xiaowujee commented Mar 11, 2021 •
edited
Loading

operator version quay.io/solo-io/wasme:0.0.32
istio version 1.8.2
kubernetes version 1.18.3

filterDeployment:

apiVersion: wasme.io/v1
kind: FilterDeployment
metadata:
  name: bookinfo-custom-demo1
  namespace: bookinfo
spec:
  deployment:
    istio:
      kind: Deployment
      labels:
        app: details
        version: v1      
  filter:
    image: webassemblyhub.io/xiaowu/wasme-test:v0.20

operator error log


time="2021-03-11T02:07:49Z" level=info msg="image is already cached" cache="{wasme-cache wasme}" image="webassemblyhub.io/xiaowu/wasme-test:v0.20"

time="2021-03-11T02:07:49Z" level=info msg="updated workload sidecar annotations" filter="id:\"bookinfo-custom-demo1.bookinfo\" image:\"webassemblyhub.io/xiaowu/wasme-test:v0.20\" rootID:\"root_id\" " workload=details-v1

time="2021-03-11T02:07:49Z" level=info msg="created Istio EnvoyFilter resource" envoy_filter_resource=details-v1-bookinfo-custom-demo1.bookinfo.bookinfo filter="id:\"bookinfo-custom-demo1.bookinfo\" image:\"webassemblyhub.io/xiaowu/wasme-test:v0.20\" rootID:\"root_id\" " workload=details-v1

{"level":"debug","ts":1615428469.5524452,"msg":"applied filter to workload","result":"state:Succeeded "}

panic: reflect: slice index out of range [recovered]

	panic: reflect: slice index out of range [recovered]

	panic: reflect: slice index out of range

goroutine 200 [running]:

k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0)

	/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/runtime/runtime.go:55 +0x105

panic(0x1d78d20, 0x2590e10)

	/opt/hostedtoolcache/go/1.14.12/x64/src/runtime/panic.go:969 +0x166

encoding/json.(*encodeState).marshal.func1(0xc001839220)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:326 +0x9a

panic(0x1d78d20, 0x2590e10)

	/opt/hostedtoolcache/go/1.14.12/x64/src/runtime/panic.go:969 +0x166

reflect.Value.Index(0x1d203e0, 0xc001040920, 0x197, 0x1, 0x2010000, 0xc00092c5a0, 0x199)

	/opt/hostedtoolcache/go/1.14.12/x64/src/reflect/value.go:946 +0x1e7

encoding/json.arrayEncoder.encode(0xc000b95620, 0xc000533200, 0x1d203e0, 0xc001040920, 0x197, 0x1d70000)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:886 +0x8d

encoding/json.sliceEncoder.encode(0xc000b92c80, 0xc000533200, 0x1d203e0, 0xc001040920, 0x197, 0x0)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:860 +0x8f

encoding/json.structEncoder.encode(0xc000292900, 0x10, 0x10, 0xc000b95650, 0xc000533200, 0x2138280, 0xc001040840, 0x199, 0x0)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:759 +0x2bb

encoding/json.structEncoder.encode(0xc000a6c900, 0x5, 0x8, 0xc000b95830, 0xc000533200, 0x1fa0f80, 0xc001040820, 0x199, 0x1e10000)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:759 +0x2bb

encoding/json.ptrEncoder.encode(0xc000b95860, 0xc000533200, 0x215e2c0, 0xc001040820, 0x16, 0x2150000)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:915 +0x116

encoding/json.(*encodeState).reflectValue(0xc000533200, 0x215e2c0, 0xc001040820, 0x16, 0xc001040000)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:358 +0x82

encoding/json.interfaceEncoder(0xc000533200, 0x20e4860, 0xc00189f580, 0x94, 0x0)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:714 +0xac

encoding/json.structEncoder.encode(0xc0008a06c0, 0x4, 0x4, 0xc000383710, 0xc000533200, 0x1fea120, 0xc00189f580, 0x99, 0x1fe0000)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:759 +0x2bb

encoding/json.(*encodeState).reflectValue(0xc000533200, 0x1fea120, 0xc00189f580, 0x99, 0xc001830000)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:358 +0x82

encoding/json.(*encodeState).marshal(0xc000533200, 0x1fea120, 0xc00189f580, 0x580000, 0x0, 0x0)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/encode.go:330 +0xf0

encoding/json.(*Encoder).Encode(0xc0011e6320, 0x1fea120, 0xc00189f580, 0x5aff8b, 0xc00112c120)

	/opt/hostedtoolcache/go/1.14.12/x64/src/encoding/json/stream.go:206 +0x8a

go.uber.org/zap/zapcore.(*jsonEncoder).encodeReflected(0xc00112c120, 0x1fea120, 0xc00189f580, 0x9dc1b6348c84dd2f, 0xffffffffffffffd2, 0x3500, 0x3438323435313631, 0x3430333337353936)

	/home/runner/go/pkg/mod/go.uber.org/[email protected]/zapcore/json_encoder.go:157 +0x63

go.uber.org/zap/zapcore.(*jsonEncoder).AddReflected(0xc00112c120, 0x219b31e, 0x5, 0x1fea120, 0xc00189f580, 0xc0014dc000, 0x16)

	/home/runner/go/pkg/mod/go.uber.org/[email protected]/zapcore/json_encoder.go:165 +0x4c

go.uber.org/zap/zapcore.Field.AddTo(0x219b31e, 0x5, 0x17, 0x0, 0x0, 0x0, 0x1fea120, 0xc00189f580, 0x2654f80, 0xc00112c120)

	/home/runner/go/pkg/mod/go.uber.org/[email protected]/zapcore/field.go:164 +0xb74

go.uber.org/zap/zapcore.addFields(0x2654f80, 0xc00112c120, 0xc000532300, 0x2, 0x2)

	/home/runner/go/pkg/mod/go.uber.org/[email protected]/zapcore/field.go:204 +0xcf

go.uber.org/zap/zapcore.(*jsonEncoder).EncodeEntry(0xc000a403c0, 0xff, 0xc00a7cbd622bef27, 0x2bdbb3326e, 0x39a10e0, 0x0, 0x0, 0x21a7856, 0xd, 0x0, ...)

	/home/runner/go/pkg/mod/go.uber.org/[email protected]/zapcore/json_encoder.go:383 +0x1e4

sigs.k8s.io/controller-runtime/pkg/log/zap.(*KubeAwareEncoder).EncodeEntry(0xc00000c1a0, 0xff, 0xc00a7cbd622bef27, 0x2bdbb3326e, 0x39a10e0, 0x0, 0x0, 0x21a7856, 0xd, 0x0, ...)

	/home/runner/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/log/zap/kube_helpers.go:128 +0x174

go.uber.org/zap/zapcore.(*ioCore).Write(0xc000a403f0, 0xff, 0xc00a7cbd622bef27, 0x2bdbb3326e, 0x39a10e0, 0x0, 0x0, 0x21a7856, 0xd, 0x0, ...)

	/home/runner/go/pkg/mod/go.uber.org/[email protected]/zapcore/core.go:86 +0xa9

go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc000981970, 0xc000532300, 0x2, 0x2)

	/home/runner/go/pkg/mod/go.uber.org/[email protected]/zapcore/entry.go:216 +0x117

github.com/go-logr/zapr.(*infoLogger).Info(0xc000025e10, 0x21a7856, 0xd, 0xc00189f5c0, 0x4, 0x4)

	/home/runner/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:70 +0xdd

github.com/solo-io/skv2/pkg/events.(*cache).handleEvent(0xc00094fc20, 0x25e41c0, 0xc00189f580, 0x26457a0, 0xc00094ff00)

	/home/runner/go/pkg/mod/github.com/solo-io/[email protected]/pkg/events/cache.go:84 +0x16d

github.com/solo-io/skv2/pkg/events.(*cache).Update(0xc00094fc20, 0x7f1b913832b0, 0xc001040820, 0x25dcf00, 0xc001040820, 0x7f1b913832b0, 0xc000190340, 0x25dcf00, 0xc000190340, 0x26457a0, ...)

	/home/runner/go/pkg/mod/github.com/solo-io/[email protected]/pkg/events/cache.go:102 +0xab

sigs.k8s.io/controller-runtime/pkg/source/internal.EventHandler.OnUpdate(0x7f1b911839b8, 0xc00094fc20, 0x26457a0, 0xc00094ff00, 0x0, 0x0, 0x0, 0x215e2c0, 0xc001040820, 0x215e2c0, ...)

	/home/runner/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/source/internal/eventsource.go:124 +0x444

k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0xc000421738, 0x2, 0x1)

	/home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:656 +0x1e8

k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc001839e18, 0x2000000a87748, 0x0)

	/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:292 +0x51

k8s.io/client-go/tools/cache.(*processorListener).run.func1()

	/home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:652 +0x79

k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0009e1760)

	/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5f

k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001839f60, 0xdf8475800, 0x0, 0x1, 0xc0004f48a0)

	/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8

k8s.io/apimachinery/pkg/util/wait.Until(...)

	/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88

k8s.io/client-go/tools/cache.(*processorListener).run(0xc000b91880)

	/home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:650 +0x9b

k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0003d9020, 0xc000b929a0)

	/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x51

created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start

	/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:69 +0x62

image
@xiaowujee xiaowujee added the question Further information is requested label Mar 11, 2021
@Sodman
Copy link
Member

Sodman commented Mar 29, 2021

Hi @xiaowujee, I haven't been able to reproduce this on my end. Does this happen consistently for you? Can you check the logs of the cache pod and see if there's anything suspicious in there? Also I would try restarting the cache pod and see if that makes any difference.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
question Further information is requested
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
2 participants
@Sodman @xiaowujee