huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

make all tests allow 'non-strict' conformance (#149)

This commit is contained in:
Andrew Stanton-Nurse 2017-03-13 15:40:55 -07:00 committed by GitHub
parent 5529cab0e3
commit 9b553ca1c0
5 changed files with 47 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -30,6 +30,6 @@ branches:
before_install: before_install:
- if test "$TRAVIS_OS_NAME" == "osx"; then brew update; brew install openssl python; ln -s /usr/local/opt/openssl/lib/libcrypto.1.0.0.dylib /usr/local/lib/; ln -s /usr/local/opt/openssl/lib/libssl.1.0.0.dylib /usr/local/lib/; fi - if test "$TRAVIS_OS_NAME" == "osx"; then brew update; brew install openssl python; ln -s /usr/local/opt/openssl/lib/libcrypto.1.0.0.dylib /usr/local/lib/; ln -s /usr/local/opt/openssl/lib/libssl.1.0.0.dylib /usr/local/lib/; fi
install: install:
- sudo pip install autobahntestsuite "six>=1.9.0" - pip install autobahntestsuite "six>=1.9.0"
script: script:
- ./build.sh - ./build.sh

82
WebSockets.sln
View File

@ -1,6 +1,6 @@
Microsoft Visual Studio Solution File, Format Version 12.00 Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15 # Visual Studio 15
VisualStudioVersion = 15.0.26123.0 VisualStudioVersion = 15.0.26228.4
MinimumVisualStudioVersion = 10.0.40219.1 MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{2C7947A5-9FBD-4267-97C1-2D726D7B3BAF}" Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{2C7947A5-9FBD-4267-97C1-2D726D7B3BAF}"
EndProject EndProject
@ -63,64 +63,64 @@ Global
{B7246F23-6A4B-492F-AB61-292AA1A9E9D5}.Release|x86.Build.0 = Release|Any CPU {B7246F23-6A4B-492F-AB61-292AA1A9E9D5}.Release|x86.Build.0 = Release|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|Any CPU.Build.0 = Debug|Any CPU {CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x64.ActiveCfg = Debug|x64 {CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x64.ActiveCfg = Debug|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x64.Build.0 = Debug|x64 {CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x64.Build.0 = Debug|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x86.ActiveCfg = Debug|x86 {CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x86.ActiveCfg = Debug|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x86.Build.0 = Debug|x86 {CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x86.Build.0 = Debug|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|Any CPU.ActiveCfg = Release|Any CPU {CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|Any CPU.Build.0 = Release|Any CPU {CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|Any CPU.Build.0 = Release|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x64.ActiveCfg = Release|x64 {CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x64.ActiveCfg = Release|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x64.Build.0 = Release|x64 {CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x64.Build.0 = Release|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x86.ActiveCfg = Release|x86 {CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x86.ActiveCfg = Release|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x86.Build.0 = Release|x86 {CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x86.Build.0 = Release|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|Any CPU.Build.0 = Debug|Any CPU {5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x64.ActiveCfg = Debug|x64 {5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x64.ActiveCfg = Debug|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x64.Build.0 = Debug|x64 {5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x64.Build.0 = Debug|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x86.ActiveCfg = Debug|x86 {5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x86.ActiveCfg = Debug|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x86.Build.0 = Debug|x86 {5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x86.Build.0 = Debug|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|Any CPU.ActiveCfg = Release|Any CPU {5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|Any CPU.Build.0 = Release|Any CPU {5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|Any CPU.Build.0 = Release|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x64.ActiveCfg = Release|x64 {5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x64.ActiveCfg = Release|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x64.Build.0 = Release|x64 {5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x64.Build.0 = Release|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x86.ActiveCfg = Release|x86 {5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x86.ActiveCfg = Release|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x86.Build.0 = Release|x86 {5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x86.Build.0 = Release|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|Any CPU.Build.0 = Debug|Any CPU {74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|Any CPU.Build.0 = Debug|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x64.ActiveCfg = Debug|x64 {74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x64.ActiveCfg = Debug|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x64.Build.0 = Debug|x64 {74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x64.Build.0 = Debug|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x86.ActiveCfg = Debug|x86 {74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x86.ActiveCfg = Debug|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x86.Build.0 = Debug|x86 {74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x86.Build.0 = Debug|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|Any CPU.ActiveCfg = Release|Any CPU {74F45408-1959-4FEE-9511-25D40F4913FD}.Release|Any CPU.ActiveCfg = Release|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|Any CPU.Build.0 = Release|Any CPU {74F45408-1959-4FEE-9511-25D40F4913FD}.Release|Any CPU.Build.0 = Release|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x64.ActiveCfg = Release|x64 {74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x64.ActiveCfg = Release|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x64.Build.0 = Release|x64 {74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x64.Build.0 = Release|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x86.ActiveCfg = Release|x86 {74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x86.ActiveCfg = Release|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x86.Build.0 = Release|x86 {74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x86.Build.0 = Release|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|Any CPU.Build.0 = Debug|Any CPU {421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x64.ActiveCfg = Debug|x64 {421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x64.ActiveCfg = Debug|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x64.Build.0 = Debug|x64 {421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x64.Build.0 = Debug|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x86.ActiveCfg = Debug|x86 {421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x86.ActiveCfg = Debug|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x86.Build.0 = Debug|x86 {421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x86.Build.0 = Debug|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|Any CPU.ActiveCfg = Release|Any CPU {421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|Any CPU.Build.0 = Release|Any CPU {421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|Any CPU.Build.0 = Release|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x64.ActiveCfg = Release|x64 {421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x64.ActiveCfg = Release|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x64.Build.0 = Release|x64 {421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x64.Build.0 = Release|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x86.ActiveCfg = Release|x86 {421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x86.ActiveCfg = Release|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x86.Build.0 = Release|x86 {421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x86.Build.0 = Release|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|Any CPU.Build.0 = Debug|Any CPU {150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x64.ActiveCfg = Debug|x64 {150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x64.ActiveCfg = Debug|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x64.Build.0 = Debug|x64 {150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x64.Build.0 = Debug|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x86.ActiveCfg = Debug|x86 {150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x86.ActiveCfg = Debug|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x86.Build.0 = Debug|x86 {150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x86.Build.0 = Debug|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|Any CPU.ActiveCfg = Release|Any CPU {150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|Any CPU.Build.0 = Release|Any CPU {150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|Any CPU.Build.0 = Release|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x64.ActiveCfg = Release|x64 {150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x64.ActiveCfg = Release|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x64.Build.0 = Release|x64 {150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x64.Build.0 = Release|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x86.ActiveCfg = Release|x86 {150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x86.ActiveCfg = Release|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x86.Build.0 = Release|x86 {150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE

11
test/Microsoft.AspNetCore.WebSockets.ConformanceTest/Autobahn/AutobahnExpectations.cs
View File

@ -21,7 +21,6 @@ namespace Microsoft.AspNetCore.WebSockets.ConformanceTest.Autobahn
public AutobahnExpectations Fail(params string[] caseSpecs) => Expect(Expectation.Fail, caseSpecs); public AutobahnExpectations Fail(params string[] caseSpecs) => Expect(Expectation.Fail, caseSpecs);
public AutobahnExpectations NonStrict(params string[] caseSpecs) => Expect(Expectation.NonStrict, caseSpecs); public AutobahnExpectations NonStrict(params string[] caseSpecs) => Expect(Expectation.NonStrict, caseSpecs);
public AutobahnExpectations OkOrNonStrict(params string[] caseSpecs) => Expect(Expectation.OkOrNonStrict, caseSpecs);
public AutobahnExpectations OkOrFail(params string[] caseSpecs) => Expect(Expectation.OkOrFail, caseSpecs); public AutobahnExpectations OkOrFail(params string[] caseSpecs) => Expect(Expectation.OkOrFail, caseSpecs);
public AutobahnExpectations Expect(Expectation expectation, params string[] caseSpecs) public AutobahnExpectations Expect(Expectation expectation, params string[] caseSpecs)
@ -61,21 +60,15 @@ namespace Microsoft.AspNetCore.WebSockets.ConformanceTest.Autobahn
} }
break; break;
case Expectation.Ok: case Expectation.Ok:
if (!caseResult.BehaviorIs("OK"))
{
failures.AppendLine($"Case {serverResult.Name}:{caseResult.Name}. Expected 'OK', but got '{caseResult.ActualBehavior}'");
}
break;
case Expectation.OkOrNonStrict:
if (!caseResult.BehaviorIs("NON-STRICT") && !caseResult.BehaviorIs("OK")) if (!caseResult.BehaviorIs("NON-STRICT") && !caseResult.BehaviorIs("OK"))
{ {
failures.AppendLine($"Case {serverResult.Name}:{caseResult.Name}. Expected 'NON-STRICT' or 'OK', but got '{caseResult.ActualBehavior}'"); failures.AppendLine($"Case {serverResult.Name}:{caseResult.Name}. Expected 'NON-STRICT' or 'OK', but got '{caseResult.ActualBehavior}'");
} }
break; break;
case Expectation.OkOrFail: case Expectation.OkOrFail:
if (!caseResult.BehaviorIs("FAILED") && !caseResult.BehaviorIs("OK")) if (!caseResult.BehaviorIs("NON-STRICT") && !caseResult.BehaviorIs("FAILED") && !caseResult.BehaviorIs("OK"))
{ {
failures.AppendLine($"Case {serverResult.Name}:{caseResult.Name}. Expected 'FAILED' or 'OK', but got '{caseResult.ActualBehavior}'"); failures.AppendLine($"Case {serverResult.Name}:{caseResult.Name}. Expected 'FAILED', 'NON-STRICT' or 'OK', but got '{caseResult.ActualBehavior}'");
} }
break; break;
default: default:

3
test/Microsoft.AspNetCore.WebSockets.ConformanceTest/Autobahn/Expectation.cs
View File

@ -5,7 +5,6 @@
Fail, Fail,
NonStrict, NonStrict,
OkOrFail, OkOrFail,
Ok, Ok
OkOrNonStrict
} }
} }

6
test/Microsoft.AspNetCore.WebSockets.ConformanceTest/AutobahnTests.cs
View File

@ -65,14 +65,12 @@ namespace Microsoft.AspNetCore.WebSockets.ConformanceTest
// IIS Express tests are a bit flaky, some tests fail occasionally or get non-strict passes // IIS Express tests are a bit flaky, some tests fail occasionally or get non-strict passes
// https://github.com/aspnet/WebSockets/issues/100 // https://github.com/aspnet/WebSockets/issues/100
await tester.DeployTestAndAddToSpec(ServerType.IISExpress, ssl: false, environment: "ManagedSockets", cancellationToken: cts.Token, expectationConfig: expect => expect await tester.DeployTestAndAddToSpec(ServerType.IISExpress, ssl: false, environment: "ManagedSockets", cancellationToken: cts.Token, expectationConfig: expect => expect
.OkOrFail(Enumerable.Range(1, 20).Select(i => $"5.{i}").ToArray()) // 5.* occasionally fail on IIS express .OkOrFail(Enumerable.Range(1, 20).Select(i => $"5.{i}").ToArray())); // 5.* occasionally fail on IIS express
.OkOrNonStrict("3.2", "3.3", "3.4", "4.1.3", "4.1.4", "4.1.5", "4.2.3", "4.2.4", "4.2.5", "5.15")); // These occasionally get non-strict results
} }
// WebListener occasionally gives a non-strict response on 3.2. IIS Express seems to have the same behavior. Wonder if it's related to HttpSys? // WebListener occasionally gives a non-strict response on 3.2. IIS Express seems to have the same behavior. Wonder if it's related to HttpSys?
// For now, just allow the non-strict response, it's not a failure. // For now, just allow the non-strict response, it's not a failure.
await tester.DeployTestAndAddToSpec(ServerType.WebListener, ssl: false, environment: "ManagedSockets", cancellationToken: cts.Token, expectationConfig: expect => expect await tester.DeployTestAndAddToSpec(ServerType.WebListener, ssl: false, environment: "ManagedSockets", cancellationToken: cts.Token);
.OkOrNonStrict("3.2", "4.1.3", "4.2.4"));
} }
} }