huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

make all tests allow 'non-strict' conformance (#149)

This commit is contained in:
Andrew Stanton-Nurse 2017-03-13 15:40:55 -07:00 committed by GitHub
parent 5529cab0e3
commit 9b553ca1c0
5 changed files with 47 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -30,6 +30,6 @@ branches:
before_install:
- if test "$TRAVIS_OS_NAME" == "osx"; then brew update; brew install openssl python; ln -s /usr/local/opt/openssl/lib/libcrypto.1.0.0.dylib /usr/local/lib/; ln -s /usr/local/opt/openssl/lib/libssl.1.0.0.dylib /usr/local/lib/; fi
install:
- sudo pip install autobahntestsuite "six>=1.9.0"
- pip install autobahntestsuite "six>=1.9.0"
script:
- ./build.sh

82
WebSockets.sln
View File

@ -1,6 +1,6 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.26123.0
VisualStudioVersion = 15.0.26228.4
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{2C7947A5-9FBD-4267-97C1-2D726D7B3BAF}"
EndProject
@ -63,64 +63,64 @@ Global
{B7246F23-6A4B-492F-AB61-292AA1A9E9D5}.Release|x86.Build.0 = Release|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x64.ActiveCfg = Debug|x64
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x64.Build.0 = Debug|x64
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x86.ActiveCfg = Debug|x86
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x86.Build.0 = Debug|x86
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x64.ActiveCfg = Debug|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x64.Build.0 = Debug|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x86.ActiveCfg = Debug|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Debug|x86.Build.0 = Debug|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|Any CPU.Build.0 = Release|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x64.ActiveCfg = Release|x64
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x64.Build.0 = Release|x64
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x86.ActiveCfg = Release|x86
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x86.Build.0 = Release|x86
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x64.ActiveCfg = Release|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x64.Build.0 = Release|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x86.ActiveCfg = Release|Any CPU
{CDE16880-0374-46FA-8896-99F1B90B4B6F}.Release|x86.Build.0 = Release|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x64.ActiveCfg = Debug|x64
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x64.Build.0 = Debug|x64
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x86.ActiveCfg = Debug|x86
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x86.Build.0 = Debug|x86
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x64.ActiveCfg = Debug|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x64.Build.0 = Debug|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x86.ActiveCfg = Debug|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Debug|x86.Build.0 = Debug|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|Any CPU.Build.0 = Release|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x64.ActiveCfg = Release|x64
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x64.Build.0 = Release|x64
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x86.ActiveCfg = Release|x86
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x86.Build.0 = Release|x86
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x64.ActiveCfg = Release|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x64.Build.0 = Release|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x86.ActiveCfg = Release|Any CPU
{5AFA74F5-9B1D-4FC5-815F-EF471F5AC1EF}.Release|x86.Build.0 = Release|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|Any CPU.Build.0 = Debug|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x64.ActiveCfg = Debug|x64
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x64.Build.0 = Debug|x64
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x86.ActiveCfg = Debug|x86
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x86.Build.0 = Debug|x86
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x64.ActiveCfg = Debug|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x64.Build.0 = Debug|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x86.ActiveCfg = Debug|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Debug|x86.Build.0 = Debug|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|Any CPU.ActiveCfg = Release|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|Any CPU.Build.0 = Release|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x64.ActiveCfg = Release|x64
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x64.Build.0 = Release|x64
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x86.ActiveCfg = Release|x86
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x86.Build.0 = Release|x86
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x64.ActiveCfg = Release|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x64.Build.0 = Release|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x86.ActiveCfg = Release|Any CPU
{74F45408-1959-4FEE-9511-25D40F4913FD}.Release|x86.Build.0 = Release|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x64.ActiveCfg = Debug|x64
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x64.Build.0 = Debug|x64
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x86.ActiveCfg = Debug|x86
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x86.Build.0 = Debug|x86
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x64.ActiveCfg = Debug|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x64.Build.0 = Debug|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x86.ActiveCfg = Debug|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Debug|x86.Build.0 = Debug|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|Any CPU.Build.0 = Release|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x64.ActiveCfg = Release|x64
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x64.Build.0 = Release|x64
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x86.ActiveCfg = Release|x86
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x86.Build.0 = Release|x86
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x64.ActiveCfg = Release|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x64.Build.0 = Release|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x86.ActiveCfg = Release|Any CPU
{421954B0-5C6B-4092-8D4D-EACA4CE60AFB}.Release|x86.Build.0 = Release|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x64.ActiveCfg = Debug|x64
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x64.Build.0 = Debug|x64
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x86.ActiveCfg = Debug|x86
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x86.Build.0 = Debug|x86
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x64.ActiveCfg = Debug|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x64.Build.0 = Debug|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x86.ActiveCfg = Debug|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Debug|x86.Build.0 = Debug|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|Any CPU.Build.0 = Release|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x64.ActiveCfg = Release|x64
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x64.Build.0 = Release|x64
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x86.ActiveCfg = Release|x86
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x86.Build.0 = Release|x86
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x64.ActiveCfg = Release|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x64.Build.0 = Release|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x86.ActiveCfg = Release|Any CPU
{150DF5A8-87C6-42F7-8886-CE07BFD02FD2}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

11
test/Microsoft.AspNetCore.WebSockets.ConformanceTest/Autobahn/AutobahnExpectations.cs
View File

@ -21,7 +21,6 @@ namespace Microsoft.AspNetCore.WebSockets.ConformanceTest.Autobahn
public AutobahnExpectations Fail(params string[] caseSpecs) => Expect(Expectation.Fail, caseSpecs);
public AutobahnExpectations NonStrict(params string[] caseSpecs) => Expect(Expectation.NonStrict, caseSpecs);
public AutobahnExpectations OkOrNonStrict(params string[] caseSpecs) => Expect(Expectation.OkOrNonStrict, caseSpecs);
public AutobahnExpectations OkOrFail(params string[] caseSpecs) => Expect(Expectation.OkOrFail, caseSpecs);
public AutobahnExpectations Expect(Expectation expectation, params string[] caseSpecs)
@ -61,21 +60,15 @@ namespace Microsoft.AspNetCore.WebSockets.ConformanceTest.Autobahn
}
break;
case Expectation.Ok:
if (!caseResult.BehaviorIs("OK"))
{
failures.AppendLine($"Case {serverResult.Name}:{caseResult.Name}. Expected 'OK', but got '{caseResult.ActualBehavior}'");
}
break;
case Expectation.OkOrNonStrict:
if (!caseResult.BehaviorIs("NON-STRICT") && !caseResult.BehaviorIs("OK"))
{
failures.AppendLine($"Case {serverResult.Name}:{caseResult.Name}. Expected 'NON-STRICT' or 'OK', but got '{caseResult.ActualBehavior}'");
}
break;
case Expectation.OkOrFail:
if (!caseResult.BehaviorIs("FAILED") && !caseResult.BehaviorIs("OK"))
if (!caseResult.BehaviorIs("NON-STRICT") && !caseResult.BehaviorIs("FAILED") && !caseResult.BehaviorIs("OK"))
{
failures.AppendLine($"Case {serverResult.Name}:{caseResult.Name}. Expected 'FAILED' or 'OK', but got '{caseResult.ActualBehavior}'");
failures.AppendLine($"Case {serverResult.Name}:{caseResult.Name}. Expected 'FAILED', 'NON-STRICT' or 'OK', but got '{caseResult.ActualBehavior}'");
}
break;
default:

3
test/Microsoft.AspNetCore.WebSockets.ConformanceTest/Autobahn/Expectation.cs
View File

@ -5,7 +5,6 @@
Fail,
NonStrict,
OkOrFail,
Ok,
OkOrNonStrict
Ok
}
}

6
test/Microsoft.AspNetCore.WebSockets.ConformanceTest/AutobahnTests.cs
View File

@ -65,14 +65,12 @@ namespace Microsoft.AspNetCore.WebSockets.ConformanceTest
// IIS Express tests are a bit flaky, some tests fail occasionally or get non-strict passes
// https://github.com/aspnet/WebSockets/issues/100
await tester.DeployTestAndAddToSpec(ServerType.IISExpress, ssl: false, environment: "ManagedSockets", cancellationToken: cts.Token, expectationConfig: expect => expect
.OkOrFail(Enumerable.Range(1, 20).Select(i => $"5.{i}").ToArray()) // 5.* occasionally fail on IIS express
.OkOrNonStrict("3.2", "3.3", "3.4", "4.1.3", "4.1.4", "4.1.5", "4.2.3", "4.2.4", "4.2.5", "5.15")); // These occasionally get non-strict results
.OkOrFail(Enumerable.Range(1, 20).Select(i => $"5.{i}").ToArray())); // 5.* occasionally fail on IIS express
}
// WebListener occasionally gives a non-strict response on 3.2. IIS Express seems to have the same behavior. Wonder if it's related to HttpSys?
// For now, just allow the non-strict response, it's not a failure.
await tester.DeployTestAndAddToSpec(ServerType.WebListener, ssl: false, environment: "ManagedSockets", cancellationToken: cts.Token, expectationConfig: expect => expect
.OkOrNonStrict("3.2", "4.1.3", "4.2.4"));
await tester.DeployTestAndAddToSpec(ServerType.WebListener, ssl: false, environment: "ManagedSockets", cancellationToken: cts.Token);
}
}